{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "732e7ed6",
   "metadata": {},
   "outputs": [],
   "source": [
    "import lightgbm as lgb\n",
    "import pandas as pd\n",
    "from sklearn import datasets\n",
    "\n",
    "from sklearn.utils import shuffle\n",
    "from sklearn import metrics\n",
    "# 画图\n",
    "import matplotlib.pyplot as plt\n",
    "# 参数搜索\n",
    "from sklearn.model_selection import GridSearchCV, RandomizedSearchCV,cross_val_score\n",
    "from bayes_opt import BayesianOptimization\n",
    "from sklearn.metrics import roc_auc_score\n",
    "import json\n",
    "import pickle\n",
    "from tqdm import tqdm\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ef1b7cf6",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "# 任务1：模型训练与预测"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "3771723f",
   "metadata": {
    "hidden": true,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sepal length (cm)</th>\n",
       "      <th>sepal width (cm)</th>\n",
       "      <th>petal length (cm)</th>\n",
       "      <th>petal width (cm)</th>\n",
       "      <th>labels</th>\n",
       "      <th>pred</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>74</th>\n",
       "      <td>6.4</td>\n",
       "      <td>2.9</td>\n",
       "      <td>4.3</td>\n",
       "      <td>1.3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>5.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.6</td>\n",
       "      <td>0.2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>86</th>\n",
       "      <td>6.7</td>\n",
       "      <td>3.1</td>\n",
       "      <td>4.7</td>\n",
       "      <td>1.5</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>82</th>\n",
       "      <td>5.8</td>\n",
       "      <td>2.7</td>\n",
       "      <td>3.9</td>\n",
       "      <td>1.2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>132</th>\n",
       "      <td>6.4</td>\n",
       "      <td>2.8</td>\n",
       "      <td>5.6</td>\n",
       "      <td>2.2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     sepal length (cm)  sepal width (cm)  petal length (cm)  petal width (cm)  \\\n",
       "74                 6.4               2.9                4.3               1.3   \n",
       "25                 5.0               3.0                1.6               0.2   \n",
       "86                 6.7               3.1                4.7               1.5   \n",
       "82                 5.8               2.7                3.9               1.2   \n",
       "132                6.4               2.8                5.6               2.2   \n",
       "\n",
       "     labels  pred  \n",
       "74        1     1  \n",
       "25        0     0  \n",
       "86        1     1  \n",
       "82        1     1  \n",
       "132       2     2  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 加载鸢尾花\n",
    "iris_data = datasets.load_iris()\n",
    "iris_df = pd.DataFrame(iris_data.data, columns=iris_data.feature_names)\n",
    "iris_df['labels'] = iris_data.target\n",
    "\n",
    "# 随机切分数据\n",
    "iris_df = shuffle(iris_df)\n",
    "train_data = iris_df[:100]\n",
    "test_data = iris_df[100:]\n",
    "\n",
    "# 建议训练\n",
    "lgb_model = lgb.LGBMClassifier()\n",
    "lgb_model = lgb_model.fit(train_data[iris_data.feature_names],train_data['labels'])\n",
    "test_y_pred = lgb_model.predict(test_data[iris_data.feature_names])\n",
    "test_data['pred'] = test_y_pred\n",
    "test_data.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "664c9ca2",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "# 任务2：模型保存与加载"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "fd6398ff",
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sepal length (cm)</th>\n",
       "      <th>sepal width (cm)</th>\n",
       "      <th>petal length (cm)</th>\n",
       "      <th>petal width (cm)</th>\n",
       "      <th>labels</th>\n",
       "      <th>pred</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>148</th>\n",
       "      <td>6.2</td>\n",
       "      <td>3.4</td>\n",
       "      <td>5.4</td>\n",
       "      <td>2.3</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>5.4</td>\n",
       "      <td>3.9</td>\n",
       "      <td>1.7</td>\n",
       "      <td>0.4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>66</th>\n",
       "      <td>5.6</td>\n",
       "      <td>3.0</td>\n",
       "      <td>4.5</td>\n",
       "      <td>1.5</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>115</th>\n",
       "      <td>6.4</td>\n",
       "      <td>3.2</td>\n",
       "      <td>5.3</td>\n",
       "      <td>2.3</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>125</th>\n",
       "      <td>7.2</td>\n",
       "      <td>3.2</td>\n",
       "      <td>6.0</td>\n",
       "      <td>1.8</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     sepal length (cm)  sepal width (cm)  petal length (cm)  petal width (cm)  \\\n",
       "148                6.2               3.4                5.4               2.3   \n",
       "5                  5.4               3.9                1.7               0.4   \n",
       "66                 5.6               3.0                4.5               1.5   \n",
       "115                6.4               3.2                5.3               2.3   \n",
       "125                7.2               3.2                6.0               1.8   \n",
       "\n",
       "     labels  pred  \n",
       "148       2     2  \n",
       "5         0     0  \n",
       "66        1     1  \n",
       "115       2     2  \n",
       "125       2     2  "
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 保存模型为pkl\n",
    "with open('lgb_model.pkl', 'wb') as f:\n",
    "    pickle.dump(lgb_model, f)\n",
    "# 保存模型为json\n",
    "model_json = lgb_model.booster_.dump_model()\n",
    "with open('lgb_model.json', 'w+') as f:\n",
    "    json.dump(model_json, f, indent=4)\n",
    "    \n",
    "# 加载预测\n",
    "with open('lgb_model.pkl', 'rb') as f:\n",
    "    pkl_model = pickle.load(f)\n",
    "test_y_pred = pkl_model.predict(test_data[iris_data.feature_names])\n",
    "test_data['pred'] = test_y_pred\n",
    "test_data.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5717be7d",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "# 任务3：分类、回归和排序任务"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "90017bcc",
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# 分类\n",
    "class_lgb = lgb.LGBMClassifier()\n",
    "# 回归\n",
    "regress_lgb = lgb.LGBMRegressor()\n",
    "# 排序\n",
    "rank_lgb = lgb.LGBMRanker()\n",
    "\n",
    "\n",
    "# 生成数据\n",
    "class_data_2 = datasets.make_classification(n_samples=5000, n_features=20,n_classes=2)\n",
    "class_data_n = datasets.make_classification(n_samples=5000, n_features=20,n_classes=6,n_informative=5)\n",
    "regress_data = datasets.make_regression(n_samples=5000, n_features=20)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "a0342529",
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] Auto-choosing col-wise multi-threading, the overhead of testing was 0.000503 seconds.\n",
      "You can set `force_col_wise=true` to remove the overhead.\n",
      "[LightGBM] [Info] Total Bins 5100\n",
      "[LightGBM] [Info] Number of data points in the train set: 4000, number of used features: 20\n",
      "[LightGBM] [Info] Start training from score 0.493500\n"
     ]
    }
   ],
   "source": [
    "# 二分类\n",
    "train_class_2_X = class_data_2[0][:4000]\n",
    "train_class_2_y = class_data_2[1][:4000]\n",
    "test_class_2_X = class_data_2[0][4000:]\n",
    "test_class_2_y = class_data_2[1][4000:]\n",
    "\n",
    "# 原生参数\n",
    "params = {  \n",
    "    'learning_rate': 0.1\n",
    "}\n",
    "# 原生版\n",
    "train_dataset_2 = lgb.Dataset(train_class_2_X, train_class_2_y)\n",
    "model = lgb.train(train_set=train_dataset_2,params=params)\n",
    "pred = model.predict(test_class_2_X) \n",
    "\n",
    "# sklearn版\n",
    "class_lgb = lgb.LGBMClassifier()\n",
    "class_lgb.fit(train_class_2_X,train_class_2_y)\n",
    "pred = model.predict(test_class_2_X) \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "7e4a8cae",
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] Auto-choosing col-wise multi-threading, the overhead of testing was 0.000495 seconds.\n",
      "You can set `force_col_wise=true` to remove the overhead.\n",
      "[LightGBM] [Info] Total Bins 5100\n",
      "[LightGBM] [Info] Number of data points in the train set: 4000, number of used features: 20\n",
      "[LightGBM] [Info] Start training from score 2.483750\n"
     ]
    }
   ],
   "source": [
    "# 多分类\n",
    "train_class_n_X = class_data_n[0][:4000]\n",
    "train_class_n_y = class_data_n[1][:4000]\n",
    "test_class_n_X = class_data_n[0][4000:]\n",
    "test_class_n_y = class_data_n[1][4000:]\n",
    "\n",
    "# 原生参数\n",
    "params = {  \n",
    "    'learning_rate': 0.05\n",
    "}\n",
    "# 原生版\n",
    "train_dataset_n = lgb.Dataset(train_class_n_X, train_class_n_y)\n",
    "model = lgb.train(train_set=train_dataset_n,params=params)\n",
    "pred = model.predict(test_class_n_X) \n",
    "# sklearn版\n",
    "class_lgb = lgb.LGBMClassifier()\n",
    "class_lgb.fit(train_class_n_X,train_class_n_y)\n",
    "pred = model.predict(test_class_n_X) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "71e79623",
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] Auto-choosing col-wise multi-threading, the overhead of testing was 0.000536 seconds.\n",
      "You can set `force_col_wise=true` to remove the overhead.\n",
      "[LightGBM] [Info] Total Bins 5100\n",
      "[LightGBM] [Info] Number of data points in the train set: 4000, number of used features: 20\n",
      "[LightGBM] [Info] Start training from score 1.521722\n"
     ]
    }
   ],
   "source": [
    "# 回归任务\n",
    "# 多分类\n",
    "train_regress_X = regress_data[0][:4000]\n",
    "train_regress_y = regress_data[1][:4000]\n",
    "test_regress_X = regress_data[0][4000:]\n",
    "test_regress_y = regress_data[1][4000:]\n",
    "\n",
    "# 原生参数\n",
    "params = {  \n",
    "    'learning_rate': 0.05\n",
    "}\n",
    "# 原生版\n",
    "train_dataset = lgb.Dataset(train_regress_X, train_regress_y)\n",
    "model = lgb.train(train_set=train_dataset,params=params)\n",
    "pred = model.predict(test_regress_X) \n",
    "# sklearn版\n",
    "class_lgb = lgb.LGBMRegressor()\n",
    "class_lgb.fit(train_regress_X,train_regress_y)\n",
    "pred = model.predict(test_regress_X) "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1af83686",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "# 任务4：模型可视化"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "dc94d0ee",
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.50.0 (20211204.2007)\n",
       " -->\n",
       "<!-- Pages: 1 -->\n",
       "<svg width=\"546pt\" height=\"104pt\"\n",
       " viewBox=\"0.00 0.00 545.59 104.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 100)\">\n",
       "<polygon fill=\"white\" stroke=\"transparent\" points=\"-4,4 -4,-100 541.59,-100 541.59,4 -4,4\"/>\n",
       "<!-- split0 -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>split0</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"168,-76 0,-76 0,-40 168,-40 168,-76\"/>\n",
       "<text text-anchor=\"start\" x=\"8\" y=\"-55.3\" font-family=\"Times New Roman,serif\" font-weight=\"bold\" font-size=\"14.00\">petal_width_(cm)</text>\n",
       "<text text-anchor=\"start\" x=\"114\" y=\"-55.3\" font-family=\"Times New Roman,serif\" font-size=\"14.00\"> ≤</text>\n",
       "<text text-anchor=\"start\" x=\"126\" y=\"-55.3\" font-family=\"Times New Roman,serif\" font-weight=\"bold\" font-size=\"14.00\">0.800</text>\n",
       "</g>\n",
       "<!-- leaf0 -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>leaf0</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"293\" cy=\"-78\" rx=\"59.59\" ry=\"18\"/>\n",
       "<text text-anchor=\"start\" x=\"255\" y=\"-75.3\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">leaf 0: </text>\n",
       "<text text-anchor=\"start\" x=\"292\" y=\"-75.3\" font-family=\"Times New Roman,serif\" font-weight=\"bold\" font-size=\"14.00\">&#45;1.126</text>\n",
       "</g>\n",
       "<!-- split0&#45;&gt;leaf0 -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>split0&#45;&gt;leaf0</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M168,-66.02C187.27,-67.88 207.52,-69.84 225.94,-71.62\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"225.68,-75.11 235.97,-72.59 226.36,-68.14 225.68,-75.11\"/>\n",
       "<text text-anchor=\"middle\" x=\"188.5\" y=\"-71.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">yes</text>\n",
       "</g>\n",
       "<!-- split1 -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>split1</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"377,-56 209,-56 209,-20 377,-20 377,-56\"/>\n",
       "<text text-anchor=\"start\" x=\"217\" y=\"-35.3\" font-family=\"Times New Roman,serif\" font-weight=\"bold\" font-size=\"14.00\">petal_width_(cm)</text>\n",
       "<text text-anchor=\"start\" x=\"323\" y=\"-35.3\" font-family=\"Times New Roman,serif\" font-size=\"14.00\"> ≤</text>\n",
       "<text text-anchor=\"start\" x=\"335\" y=\"-35.3\" font-family=\"Times New Roman,serif\" font-weight=\"bold\" font-size=\"14.00\">1.750</text>\n",
       "</g>\n",
       "<!-- split0&#45;&gt;split1 -->\n",
       "<g id=\"edge4\" class=\"edge\">\n",
       "<title>split0&#45;&gt;split1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M168,-49.98C178.13,-49 188.52,-48 198.78,-47.01\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"199.36,-50.47 208.98,-46.02 198.69,-43.5 199.36,-50.47\"/>\n",
       "<text text-anchor=\"middle\" x=\"188.5\" y=\"-51.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">no</text>\n",
       "</g>\n",
       "<!-- leaf1 -->\n",
       "<g id=\"node4\" class=\"node\">\n",
       "<title>leaf1</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"477.79\" cy=\"-58\" rx=\"59.59\" ry=\"18\"/>\n",
       "<text text-anchor=\"start\" x=\"439.79\" y=\"-55.3\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">leaf 1: </text>\n",
       "<text text-anchor=\"start\" x=\"476.79\" y=\"-55.3\" font-family=\"Times New Roman,serif\" font-weight=\"bold\" font-size=\"14.00\">&#45;0.852</text>\n",
       "</g>\n",
       "<!-- split1&#45;&gt;leaf1 -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>split1&#45;&gt;leaf1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M377.11,-47.09C388.59,-48.35 400.24,-49.62 411.35,-50.84\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"411.04,-54.33 421.36,-51.93 411.8,-47.37 411.04,-54.33\"/>\n",
       "<text text-anchor=\"middle\" x=\"397.5\" y=\"-52.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">yes</text>\n",
       "</g>\n",
       "<!-- leaf2 -->\n",
       "<g id=\"node5\" class=\"node\">\n",
       "<title>leaf2</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"477.79\" cy=\"-18\" rx=\"59.59\" ry=\"18\"/>\n",
       "<text text-anchor=\"start\" x=\"439.79\" y=\"-15.3\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">leaf 2: </text>\n",
       "<text text-anchor=\"start\" x=\"476.79\" y=\"-15.3\" font-family=\"Times New Roman,serif\" font-weight=\"bold\" font-size=\"14.00\">&#45;1.115</text>\n",
       "</g>\n",
       "<!-- split1&#45;&gt;leaf2 -->\n",
       "<g id=\"edge3\" class=\"edge\">\n",
       "<title>split1&#45;&gt;leaf2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M377.11,-28.91C388.59,-27.65 400.24,-26.38 411.35,-25.16\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"411.8,-28.63 421.36,-24.07 411.04,-21.67 411.8,-28.63\"/>\n",
       "<text text-anchor=\"middle\" x=\"397.5\" y=\"-30.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">no</text>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<graphviz.graphs.Digraph at 0x29b0ec38bb0>"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import os\n",
    "os.environ[\"PATH\"] += os.pathsep + 'D:/env/Graphviz/bin'\n",
    "\n",
    "import pydotplus\n",
    "from IPython.display import Image\n",
    "\n",
    "lgb.create_tree_digraph(lgb_model, tree_index=1)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "251643f4",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "# 任务5：模型调参（网格、随机、贝叶斯）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "6a5df62d",
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd, numpy as np, time\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "# 读取数据\n",
    "data = pd.read_csv(\"https://cdn.coggle.club/kaggle-flight-delays/flights_10k.csv.zip\")\n",
    "\n",
    "# 提取有用的列\n",
    "data = data[[\"MONTH\",\"DAY\",\"DAY_OF_WEEK\",\"AIRLINE\",\"FLIGHT_NUMBER\",\"DESTINATION_AIRPORT\",\n",
    "                 \"ORIGIN_AIRPORT\",\"AIR_TIME\", \"DEPARTURE_TIME\",\"DISTANCE\",\"ARRIVAL_DELAY\"]]\n",
    "data.dropna(inplace=True)\n",
    "\n",
    "# 筛选出部分数据\n",
    "data[\"ARRIVAL_DELAY\"] = (data[\"ARRIVAL_DELAY\"]>10)*1\n",
    "\n",
    "# 进行编码\n",
    "cols = [\"AIRLINE\",\"FLIGHT_NUMBER\",\"DESTINATION_AIRPORT\",\"ORIGIN_AIRPORT\"]\n",
    "for item in cols:\n",
    "    data[item] = data[item].astype(\"category\").cat.codes +1\n",
    "\n",
    "# 划分训练集和测试集\n",
    "train, test, y_train, y_test = train_test_split(data.drop([\"ARRIVAL_DELAY\"], axis=1), data[\"ARRIVAL_DELAY\"], random_state=10, test_size=0.25)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "2329c2a5",
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['MONTH', 'DAY', 'DAY_OF_WEEK', 'AIRLINE', 'FLIGHT_NUMBER',\n",
       "       'DESTINATION_AIRPORT', 'ORIGIN_AIRPORT', 'AIR_TIME', 'DEPARTURE_TIME',\n",
       "       'DISTANCE', 'ARRIVAL_DELAY'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "id": "6595d254",
   "metadata": {
    "hidden": true,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "使用不同的深度训练: 100%|████████████████████████████████████████████████████████████████| 4/4 [00:00<00:00, 24.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3深度的acc：0.5986623127742196\n",
      "5深度的acc：0.6188442432800445\n",
      "6深度的acc：0.6257575184494091\n",
      "9深度的acc：0.6272504917040698\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "train['AIRLINE'] = train['AIRLINE'].astype('category')\n",
    "test['AIRLINE'] = test['AIRLINE'].astype('category')\n",
    "\n",
    "for i in tqdm([3,5,6,9], desc='使用不同的深度训练'):\n",
    "    # 创建模型\n",
    "    model = lgb.LGBMClassifier(max_depth=i)\n",
    "    # 训练模型\n",
    "    model = model.fit(train, y_train)\n",
    "    # 预测模型\n",
    "    test_y_pred = model.predict(test)\n",
    "    print('{}深度的acc：{}'.format(i, metrics.roc_auc_score(y_test,test_y_pred)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "30d40a90",
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "最优分数:0.8040026443954955 最优参数:{'learning_rate': 0.01, 'max_depth': 5, 'n_estimators': 1000, 'num_leaves': 128}\n"
     ]
    }
   ],
   "source": [
    "# 网格搜索\n",
    "parameters = {'max_depth': [3,5,6,9],\n",
    "             'learning_rate': [0.01, 0.005, 0.001],\n",
    "             'num_leaves': [128,256],\n",
    "             'n_estimators': [500, 1000]}\n",
    "\n",
    "model = lgb.LGBMClassifier()\n",
    "clf = GridSearchCV(model, parameters)\n",
    "clf.fit(train, y_train)\n",
    "print('最优分数:{} 最优参数:{}'.format(clf.best_score_,clf.best_params_))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "aef0bdbf",
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "最优分数:0.8031695051713591 最优参数:{'num_leaves': 256, 'n_estimators': 500, 'max_depth': 6, 'learning_rate': 0.01}\n"
     ]
    }
   ],
   "source": [
    "# 随机搜索\n",
    "parameters = {'max_depth': [3,5,6,9],\n",
    "             'learning_rate': [0.01, 0.005, 0.001],\n",
    "             'num_leaves': [128,256],\n",
    "             'n_estimators': [500, 1000]}\n",
    "\n",
    "model = lgb.LGBMClassifier()\n",
    "clf = RandomizedSearchCV(model, parameters, random_state=2019)\n",
    "clf.fit(train, y_train)\n",
    "print('最优分数:{} 最优参数:{}'.format(clf.best_score_,clf.best_params_))\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d5ae3bb1",
   "metadata": {},
   "source": [
    "# 任务6：模型微调与参数衰减"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "b9a4653b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "acc：0.7048493788558845\n",
      "acc：0.7261371181938912\n",
      "acc：0.7227440449173769\n",
      "acc：0.7197743834787432\n",
      "acc：0.7147007959722292\n",
      "acc：0.727509792055407\n",
      "acc：0.7343474204448031\n"
     ]
    }
   ],
   "source": [
    "# 读取数据\n",
    "data = pd.read_csv(\"https://cdn.coggle.club/kaggle-flight-delays/flights_10k.csv.zip\")\n",
    "\n",
    "# 提取有用的列\n",
    "data = data[[\"MONTH\",\"DAY\",\"DAY_OF_WEEK\",\"AIRLINE\",\"FLIGHT_NUMBER\",\"DESTINATION_AIRPORT\",\n",
    "                 \"ORIGIN_AIRPORT\",\"AIR_TIME\", \"DEPARTURE_TIME\",\"DISTANCE\",\"ARRIVAL_DELAY\"]]\n",
    "data.dropna(inplace=True)\n",
    "\n",
    "# 筛选出部分数据\n",
    "data[\"ARRIVAL_DELAY\"] = (data[\"ARRIVAL_DELAY\"]>10)*1\n",
    "\n",
    "# 进行编码\n",
    "cols = [\"AIRLINE\",\"FLIGHT_NUMBER\",\"DESTINATION_AIRPORT\",\"ORIGIN_AIRPORT\"]\n",
    "for item in cols:\n",
    "    data[item] = data[item].astype(\"category\").cat.codes +1\n",
    "    \n",
    "\n",
    "\n",
    "# 划分训练集和测试集\n",
    "train, test, y_train, y_test = train_test_split(data.drop([\"ARRIVAL_DELAY\"], axis=1), data[\"ARRIVAL_DELAY\"], random_state=10, test_size=0.25)\n",
    "\n",
    "\n",
    "\n",
    "model = None # 模型\n",
    "data_num = 1000 # 数据切点\n",
    "# lgb参数\n",
    "params = {\n",
    "        'task': 'train',\n",
    "        'application': 'regression',  # 目标函数\n",
    "        'boosting_type': 'gbdt',  # 设置提升类型\n",
    "        'learning_rate': 0.01,  # 学习速率\n",
    "        'num_leaves': 50,  # 叶子节点数\n",
    "        'tree_learner': 'serial',\n",
    "        'min_data_in_leaf': 100,\n",
    "        'metric': ['l1', 'l2', 'rmse'],  # l1:mae, l2:mse  # 评估函数\n",
    "        'max_bin': 255,\n",
    "        'num_trees': 300,\n",
    "'verbose': -1}\n",
    "\n",
    "for i in range(int(len(train)/data_num)):\n",
    "\n",
    "    input_X = train[i*data_num:(i+1)*data_num]\n",
    "    input_y = y_train[i*data_num:(i+1)*data_num]\n",
    "    \n",
    "    lgb_train = lgb.Dataset(input_X, input_y)\n",
    "    \n",
    "    model = lgb.train(params,\n",
    "                        lgb_train,\n",
    "                        num_boost_round=1000,\n",
    "                        keep_training_booster=True,# 增量训练\n",
    "                        init_model=model  ) # 如果gbm不为None，那么就是在上次的基础上接着训练 \n",
    "\n",
    "    # 预测模型\n",
    "    test_y_pred = model.predict(test)\n",
    "    print('acc：{}'.format(metrics.roc_auc_score(y_test,test_y_pred)))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "40b12b54",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "# 任务7：特征筛选方法"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "a00b8c9b",
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 1.0, 'Important Features')"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAegAAAEWCAYAAACtyARlAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAA99klEQVR4nO3de3wV1bn/8c9DAgpEQeRSEC0gltAYiEhBKkooonKpSrEoRSpeivTUalW8FEXoOSpYoQJHK6VHxfJr8Y5Ka7Vqu1W8i0QQFG9EBFFRBAwGSMLz+2MmcbPZuZNkNnzfr9e8mFlrzaxnBuXZs9bsPebuiIiISLQ0augAREREZE9K0CIiIhGkBC0iIhJBStAiIiIRpAQtIiISQUrQIiIiEaQELSIiEkFK0CL7GTPLN7OTGjoOADOLmdmFe/F4FZ6bmeWa2S4zK4hbFteyz1wzW1ebY4gkk97QAYjI/sfMDLAG6v4Td+/YQH3vwczS3b24oeOQ6NEdtMh+zMzGmdkLZnarmW02sw/N7Idh+cdm9rmZnRvXfr6ZzTWzp8zsazN71sy+G1f/QzN7zcy2hH/+MK4uZmY3mtkLwDfAAuAE4LbwTva2sN3ssO+tZrbUzE6IO8ZUM7vfzP4S9r/SzHqHdQuAI4DF4fGuqua1OM7MXgyvw5tmlhtXd56ZvR32+aGZXRSWNwf+CXSIuyPvEF6nG+L23+0uO7zTv9rMlgPbzCy9kv7Hhf1+bWZrzGxMdc5NUpS7a9GiZT9agHzgpHB9HFAMnAekATcAa4HbgQOAk4GvgYyw/fxw+8SwfjawJKxrBXwFjCUYnRsdbh8a1sfCY2eF9Y3DsgsT4jsHODRscwXwKXBgWDcV2A4MDeOdBryc7NzKOfdcYF2S8sOAL8PjNgIGh9ttwvphwJEEd/0DCD5g9CrvmOF1uqG8fsM484DDgaYV9Q80B7YC3cJ92wNZDf3fkZa6X3QHLSJr3P1udy8B7iNIGv/t7jvc/V/ATqBrXPt/uPtz7r4DuBboZ2aHEySx99x9gbsXu/tC4B3gx3H7znf3lWF9UbJg3P3/ufuXYZuZBB8EusU1WeLuj4fxLgB6VvN8O4R3qaXLKIIPBY+Hx93l7k8BrxMkTNz9H+7+gQeeBf5FcPdfG3Pc/WN3L6ysf2AXcLSZNXX3De6+spZ9SwpQghaRz+LWCwHcPbEsI27749IVdy8ANgEdwuWjhGN/RHB3uMe+5TGzK8Lh5C1mthloAbSOa/Jp3Po3wIFmVp3naT5x95Zxy/3Ad4GfxiduoD/B3SpmNsTMXjazTWHd0ISYaiL+WpTbv7tvA84CJgAbzOwfZpZZy74lBShBi0h1HV66YmYZBEPbn4TLdxPaHgGsj9tOfH3ebtvhfPPVwCjgEHdvCWyh6g+U1fT1fB8DCxISd3N3n25mBwAPATOAdmFMj8fFlKzPbUCzuO3vVBJruf0DuPuT7j6Y4APDO8Cfa3iekkKUoEWkuoaaWX8zawL8D/CKu39MkLS+Z2Y/Cx96Ogv4PvD3Co71GdAlbvsggjnxjUC6mV0PHFyN2BKPV1X/D/ixmZ1iZmlmdmD4YFdHoAnBMPtGoNjMhhDMzcf3eaiZtYgryyO4Tq3M7DvAb2rav5m1M7PTwgfSdgAFQEkNzlFSjBK0iFTX34ApBEPbxwJjANz9S2A4wYNdXwJXAcPd/YsKjjUbONPMvjKzOcCTBE9Fv0swPL6dKgyLx5kGXBcOE0+s6k7hB4zTgUkEifhj4Eqgkbt/DVwC3E/w0NvPgMfi9n0HWAh8GPbbgWBu/E2Ch8H+RTC3X6P+w+UKghGKTQQPqf1XVc9NUpe513RESET2N2Y2n+Bp5OsaOhaRfZ3uoEVERCJICVpERCSCNMQtIiISQbqDFhERiSC9LENqrGXLlt61a9fKG0bMtm3baN68eUOHUSOpGnuqxg2KvSGkatxQeexLly79wt3bVOVYStBSY+3ateP1119v6DCqLRaLkZub29Bh1Eiqxp6qcYNibwipGjdUHruZJf7aXrk0xC0iIhJBStAiIiIRpAQtIiISQUrQIiIiEaQELSIiEkFK0CIiIhGkBC0iIhJBStAiIiIRpAQtIiISQUrQIiIiEaQELSIiEkFK0CIiIhGkBC0iIhJBStAiIiIRpAQtIiISQUrQIiIiEaQELSIiEkFK0CIiIhGkBC0iIhJBStAiIiIRpAQtIiISQUrQIiKyX9q+fTt9+vShZ8+eZGVlMWXKFACuvPJKMjMz6dGjByNGjGDz5s1l+yxfvpx+/fqRlZVFdnY227dvB2Dnzp2MHz+esWPHkpmZyUMPPVTr+JSgRURkv3TAAQfw73//mzfffJO8vDyeeOIJXn75ZQYPHsxbb73F8uXL+d73vse0adMAKC4u5pxzzmHu3LmsXLmSWCxG48aNAbjxxhtp27YtCxYsYNWqVQwYMKDW8aXX+ggRYGYlwAqgMVAM3APMcvddZpYLPAqsidtlors/HbdfOvA2cK67f2Nm6cCnwJ/d/bdx/cSA9sB2YCfwi3A5HmgCdAZWh81vAC4O+3o93L8T8Hd3Pzourg+BpmH5xLDdOOAWYH1czD9z91UJ550NLAg3jwC2hMsXwIUJff0HuNDd7wz3PQZ4A7jS3WeY2XxgQLg/wDfu/sPkVzxQWFRCp2v+UVGTSLoiu5hxKRg3pG7sqRo3KPaGUB9x508fhpmRkZEBQFFREUVFRZgZJ598clm74447jgcffBCAf/3rX/To0YOePXsCcOihh5a1u+uuu3jnnXd47bXXaNSoEa1bt651jPvKHXShu+e4exYwGBgKTImrfz6sL12eTtjvaIKEOyEsP5kg0Y4yM0voa4y79wT+CNzi7r9y95ywzw/i+niwCnE/7+7HAMcAw83s+Li6+xJiXpW4s7uvKK0HHiNItjnuflKSvlYAZ8Vtnw28mdDmyrj+KkzOIiL7gpKSEnJycmjbti2DBw+mb9++u9XfddddDBkyBIB3330XM+OUU06hV69e/P73vwcoGwKfPHky48eP56c//SmfffZZrWPbVxJ0GXf/HBgPXJwkuVbkeaBruD4amA2sBY4rp/1LwGE1jTOeuxcCeXvreOVYCxxoZu3C63Iq8M867E9EJPLS0tLIy8tj3bp1vPrqq7z11ltldTfeeCPp6emMGTMGCIa4lyxZwl//+leWLFnCokWLeOaZZyguLmbdunUcf/zxzJs3j379+jFx4sRax7ZPDHEncvcPzawR0DYsOsHM8uKajHT3D0o3wiHtIcATZtYUGARcBLQkSNYvJenmVOCRvRGvmR0CHAU8F1d8lpn1j9vuFyby2ngQ+CmwjGB4e0dC/S1mdl24vtLdxySJdTzBByBat27D9dnFtQyp/rVrGgyhpaJUjT1V4wbF3hDqI+5YLLZHWadOnbj99ts566yzeOKJJ1i8eDEzZ87k2WefBWDr1q1069atLIl3796dBx54gEaNGnHggQdyyCGHUFBQQMeOHZkzZ07SPqpjn0zQofi75+fdfXiSNk3jEvfzwJ3A6cB/wrnoh4DJZnaZu5eE7f5qZs2BNKBXJTF4JWUnmNlyoBsw3d0/jau7z90vruT41XU/cB+QCSwEEoexr6xsaN7d5wHzAI7o0tVnrki9/4SuyC4mFeOG1I09VeMGxd4Q6iPu/DG5bNy4kcaNG9OyZUsKCwuZPHkyV199Ndu3b+exxx7j2WefpU2bNmX79OzZk0GDBtGnTx+aNGnCDTfcwGWXXcbAgQM5/fTTAcjIyOCLL77gBz/4Abm5ubWKMfX+5qrAzLoAJcDnQPcKmhaG87fx+44Gjjez/LDoUGAgUDpvPYZg7nY6cDvwkwqO/yVwSNx2K4IHuEo97+7Dzex7wBIzW+TueRUcr1bc/VMzKyKYp7+UPRN0tTRtnMbq6cP2Smz1KRaLkT8mt6HDqJFUjT1V4wbF3hDqK+4NGzZw7rnnUlJSwq5duxg1ahTDhw+na9eu7Nixg8GDBwPBg2Jz587lkEMO4fLLL+cHP/gBZsbQoUMZNiz4N/Dmm29m7NixrF+/nk6dOnH33XfXOr59LkGbWRtgLnCbu3t1pqHN7GCgP3C4u+8Iy84jGOYuTdC4e1E4FPyBmXV397fLOWQMOMfMnnZ3B84leJp6N+7+rplNA64O+6pL1wNt3b2kelP0IiL7lh49erBs2bI9yt9///1y9znnnHM455xz9ij/7ne/y3PPPUcsFqv1nXOpfeUhsaZmlmdmKwkS6b+A38XVnxDWly5nlnOcnwD/Lk3OoUeB08zsgPiG4XzwTKCiJwHmAV8Db5rZm0AGMKOctnOBE82sc7h9VkLMe+Wpand/0d0fKaf6loQ+m+yNPkVEpPr2iTtod0+roC4GtCinLiNhez4wP6FsE1A6CZGbUDczbj0fODqhfifBd6HLiysWt13It09xr0mMozLuPi5huyyexL7i2kwtb38REWlY+8odtIiIyD5ln7iD3h8k/GpYqR3u3jdZexERSW1K0CnC3VcAOQ0dh4iI1A8NcYuIiESQErSIiEgEKUGLiIhEkBK0iIhIBClBi4iIRJAStIiISAQpQYuIiESQErSIiEgEKUGLiIhEkBK0iIhIBClBi4iIRJAStIiISAQpQYuIiESQErSIiEgEKUGLiMg+Yfv27fTp04eePXuSlZXFlClTAHjggQfIysqiUaNGvP7662XtX331VXJycsjJyaFnz54sWrSorG7hwoVkZ2fTo0cPTj31VL744ot6Px9z93rvtD6YWUfgduD7BB9E/g5cCfwQeBT4EGgK/N3dJ4b7jAN6u/vF4fY5wFVAGlAMvAZMdPfNZhYL1183s3xgqbuPDPc7Exju7uMqifFRoK2794srmwoUuPsMM5sPDAC2AAZc7u7PhO1iQHtgO1AAnO/uq82sCfB74MfALmAV8Ct3XxfuVwKsIHgX+BpgLPAkcADQKrwm68NwznD3/PLiP6JLV280anZFpxhJV2QXM3NFar4KPVVjT9W4QbE3hJrEnT99GO7Otm3byMjIoKioiP79+zN79mxatGhBo0aNuOiii5gxYwa9e/cG4JtvvqFJkyakp6ezYcMGevbsySeffAJAhw4dWLVqFa1bt+aqq66iWbNmTJ06tdI4YrEYubm55dab2VJ3712Vc9on76DNzICHgUfc/Sjge0AGcGPY5Hl3PwY4BhhuZscnOcapwGXAEHfPAnoBLwLtyum2t5llVSPGluExW5pZ5wqaXunuOcBvgLkJdWPcvSdwD3BLWHYTcBDwvfDcHwEeDq8JQKG757j70cAmguTdN+zjeuC+sD6nouQsIhI1ZkZGRgYARUVFFBUVYWZ0796dbt267dG+WbNmpKcHHwS2b99O6T+T7l6W7N2drVu30qFDh/o7kdA+maCBHwHb3f1uAHcvIUi25wPNShu5eyGQBxyW5BjXEtwhry89hrvf5e6ry+lzBjCpGjGOBBYD9wJnV6H9S+XECfAc0NXMmgHnAZeF50x4DXYQXJPqHFNEJOWUlJSQk5ND27ZtGTx4MH379q2w/SuvvEJWVhbZ2dnMnTuX9PR0GjduzB133EF2dnbZnfQFF1xQT2fwrdQb+6iaLGBpfIG7bzWztUDX0jIzOwQ4iiDBJTvGG9Xo837gv8ysa6UtA6OB3wGfAQ8C0yppfyrB3XAyPyYYtu4KrHX3rQn1rxOczzOlBWaWBgwC7qxivKX7jQfGA7Ru3Ybrs4urs3sktGsaDKGlolSNPVXjBsXeEGoSdywWK1ufNWsWBQUFTJ48mczMTDp3DgYpN2/ezNKlSykoKNht39tvv52PPvqISZMm0bx5cxo1asRNN93EHXfcQYcOHZgzZw7jx49n7NixlcZRUFCwWyy1sa8maAOSTa6Xlp9gZsuBbsB0d/+0woOZZQMLCIaOJ7n7fUmalRAMM/8W+Gclx2tHkEyXuLubWbGZHe3ubyVpfouZ/R5oCxyXUPdXMysE8oFfE8whV3TeAE3NLA/oRPAh5qmKYk3k7vOAeRDMQe8v81tRkaqxp2rcoNgbQo3moMfk7lG2dOlSvvzyS8477zwAWrZsybHHHls2B51o/vz5tGrVCnfnkEMOYcyYMQCkpaUxffr0CueWS1U2B10dqfc3VzUrCYaQy5jZwcDhwAcEc9DDzex7wBIzW+TueUmO0Qv4j7uvAHLM7DaCh6jKs4AgQa+sJL6zgEOANeGcx8EEw9zXJWl7JcF8+iUEc83HxtWNcfeyRxLNbBPwXTM7yN2/jmvXi2A4HcI5aDNrQfDg3K+AOZXEm1TTxmmsnj6sJrs2qFgslvR/5lSQqrGnatyg2BtCTePeuHEjjRs3pmXLlhQWFvL0009z9dVXl9t+zZo1HH744aSnp/PRRx+xevVqOnXqxM6dO1m1ahUbN26kTZs2PPXUU3Tv3r0WZ1Qz++oc9DNAMzP7OZQN584E5gPflDZy93cJhpaT/Q1OA2aET4OXqig54+5FwK0ED3RVZDRwqrt3cvdOBEm33Hlod98FzAYamdkpFbTbRpDE/xCeM+E1aAb8O6HtFoKkP9HMGlcSr4hI5G3YsIGBAwfSo0cPfvCDHzB48GCGDx/OokWL6NixIy+99BLDhg3jlFOCf0aXLFlCz549ycnJYcSIEfzxj3+kdevWdOjQgSlTpnDiiSfSo0cP8vLymDSpOo8Y7R375B10OGw8AvijmU0m+CDyOMFDXP0Sms8lSFKdE47xuJm1Af4ZJrvNwFsEX0mqyJ0kvxMGwMw6AUcAL8f1tcbMtppZuU8zhOd0A8HXviqK4bcED6y9a2a7gHeAEZ7k+3TuvszM3iT4cLCgwrMSEYm4Hj16sGzZsj3KR4wYwYgRI/YoHzt2bLnzyhMmTGDChAl7Pcbq2CcTNIC7f0zw8FSiWLiUtivk2yeZ1xDcZZfW3UNwR5rs+Llx653i1ncA5T6PH351aY8np929V7j6SlzZuIQ2DwEPJfaf0GYHwXz0r8upz0jY/nHc+nzizl9ERBrOvjrELSIiktL22TvoKDCz84BLE4pfcPdfNUQ8IiKSOpSg61D4IyF3N3QcIiKSejTELSIiEkFK0CIiIhGkBC0iIhJBStAiIiIRpAQtIiISQUrQIiIiEaQELSIiEkFK0CIiIhGkBC0iIhJBStAiIiIRpAQtIiISQUrQIiIiEaQELSIiEkFK0CIiIhGkBC0iIilv+/bt9OnTh549e5KVlcWUKVMAeOCBB8jKyqJRo0a8/vrrZe1fffVVcnJyyMnJoWfPnixatAiAb775hmHDhpGZmUlWVhbXXHNNg5wPgLl7g3Uuqe2ILl290ajZDR1GtV2RXczMFan5KvRUjT1V4wbF3hCqG3f+9GG4O9u2bSMjI4OioiL69+/P7NmzadGiBY0aNeKiiy5ixowZ9O7dGwgScZMmTUhPT2fDhg307NmTTz75hJ07d/LKK68wcOBAdu7cyaBBg5g0aRJDhgypUiyxWIzc3Nxy681sqbv3rsqx9sk7aDMrMbO8uKWTmeWa2d+TtI2ZWe9wPcPM7jCzD8xsmZktNbNfhHWdzOythH2nmtlEM7s97GeVmRXG9XtmOfHNN7P1ZnZAuN3azPLD9T3iDNufGRfvWjOzuPpHzKwgLs7SGN40sxfNrFvcsbckXJuTEq7ZW2a22Mxa1uzqi4jUPzMjIyMDgKKiIoqKijAzunfvTrdu3fZo36xZM9LTgw8B27dvp/Sf1GbNmjFw4EAAmjRpQq9evVi3bl09ncXu9skEDRS6e07ckl/F/f4P+Ao4yt2PAU4FWlW2k7v/yt1zgKHAB3H9PljBbiXA+VWMK9Fm4HiAMJG2T6gvjaEncA8wKa7u+YRr83RYXnrNjgY2Ab+qYWwiIg2ipKSEnJwc2rZty+DBg+nbt2+F7V955RWysrLIzs5m7ty5ZQm71ObNm1m8eDGDBg2qy7DLlXpjH3XEzI4E+gA/c/ddAO6+Ebi5jrqcBVxmZn+uwb73AmcDS4CfAA8DWeW0PZjgQ0d1vAT0SFZhZuOB8QCtW7fh+uziah664bVrGgyhpaJUjT1V4wbF3hCqG3csFitbnzVrFgUFBUyePJnMzEw6d+4MBMl26dKlFBQU7Lbv7bffzkcffcSkSZNo3rw5TZo0AYJkP2nSJIYOHcratWtZu3ZtlWIpKCjYLZ7a2FcTdFMzywvX17j7iCrskwW8WZqcy3Fk3HEBvgPMqFmIrCVIsGOBxdXc9xngz2aWRpCoxwOTk8R5ENAMiP8YeULCOYx09w9KN8JjDgLuTNaxu88D5kEwB70/zG9FSarGnqpxg2JvCNWegx6Tu0fZ0qVL+fLLLznvvPMAaNmyJccee2zZHHSi+fPn06pVq7L6888/n759+zJnzpxqxV7ZHHR17A9D3FVJznsws2vDOdlP4orjh69zgLm1jPMm4Ep2/3so76m9+PISguR+FtA0yRB+aZxHAr8hTKihxCHu0uRc+qHmS4Jh/adqcD4iIg1i48aNbN68GYDCwkKefvppMjMzy22/Zs0aiouDu/SPPvqI1atX06lTJwCuu+46tmzZwqxZs+o46oql3kerurMK6Glmjdx9l7vfCNxY+vBVXXD398OkOCqu+EvgkISmrYAvEsruBRYBUyvp5jHg7iqEU+juOWbWAvg7wRx0hR8dmzZOY/X0YVU4dLTEYrGkn7hTQarGnqpxg2JvCDWJe8OGDZx77rmUlJSwa9cuRo0axfDhw1m0aBG//vWv2bhxI8OGDSMnJ4cnn3ySJUuWMH36dBo3bkyjRo344x//SOvWrVm3bh033ngjmZmZ9OrVC4CLL76YCy+8sA7OtGJK0KEwWb4O3GBmk929xMwOBKyyfWvpRuAfcdvvAR3MrLu7v21m3wV6AnkJ+z0PTAMWVnL8/sAHlbQp4+5bzOwS4FEzu8Pdi6q6r4hIQ+nRowfLli3bo3zEiBGMGLHnQOrYsWMZO3bsHuUdO3YkKl8/3t8S9CAzi39e/qcJ9RcCtwDvm9kmoBC4ui4DcveVZvYG0Cvc3mFm5wB3hx8QioAL3X1Lwn5O+fPfpXPQBuwkOK9SiXPQNyQ+be7uy8zsTYL57QU1PjkREamxfTJBu3tGkrIY0DRJ89y4NluBi8o5Zj5wdELZ1MralHOscQnbP0nYfgE4rpx9c8spz4iLIdl5ll6DFhXtH7f942TtRESkfuyrD4mJiIiktColaDM7Mu5Xr3LN7BL90lTl4n5hLH45r6HjEhGR6KvqEPdDQG8z60rw/djHgL8R/HKWlMPd9WtcIiJSI1Ud4t7l7sXACGCWu1/Gnj8vKSIiIntJVRN0kZmNBs4l+I4sQOO6CUlERESqmqDPA/oBN7r7GjPrDPy/ugtLRERk/1alOWh3X2VmVwNHhNtrgOl1GZiIiMj+rKpPcf+Y4Jesngi3c8zssTqMS0REZL9W1SHuqQSvYtwM4O55QOc6iUhERESqnKCLE39qkvLfuiQiIiK1VNXvQb9lZj8D0szsKOAS4MW6C0tERGT/VtU76F8DWcAOgh8o2ULwnmERERGpA5XeQZtZGvCYu58EXFv3IYmIiEild9DuXgJ8Y2ZJ34IkIiIie19V56C3AyvM7ClgW2mhu19SJ1GJiIjs56qaoP8RLiIiIlIPqvSQmLvfk2yp6+BERGT/tH37dvr06UPPnj3JyspiypQpAGzatInBgwdz1FFHMXjwYL766quyfaZNm0bXrl3p1q0bTz75JADffPMNw4YNIzMzk6ysLK655poGOZ+aMPfKv85sZmtI8r1nd+9SF0FJajiiS1dvNGp2Q4dRbVdkFzNzRVUHj6IlVWNP1bhBsTeE+ac2Z8CAAWzbto2MjAyKioro378/s2fP5uGHH6ZVq1Zcc801TJ8+na+++oqbb76ZVatWMXr0aF599VU++eQTTjrpJN5991127NjBK6+8wsCBA9m5cyeDBg1i0qRJDBkypE5ij8Vi5ObmlltvZkvdvXdVjlXVr1n1Bn4QLicAc9DLMmrEzEaYmZtZZrjdyczeCtdzzWyLmS0zs3fMbEYFxznPzPLCZaeZrQjXp5vZODO7LWw3Neyva9y+l4VlvcPt/Lj988xsTt1eBRGRipkZGRkZABQVFVFUVISZ8eijj3LuuecCcO655/LII48A8Oijj3L22WdzwAEH0LlzZ7p27cqrr75Ks2bNGDhwIABNmjShV69erFu3rkHOqbqqOsT9Zdyy3t1nAT+q29D2WaOBJcDZ5dQ/7+7HAMcAw83s+GSN3P1ud89x9xzgE2BguJ1s/GZFQn9nAqsS2pTun6OH/0QkCkpKSsjJyaFt27YMHjyYvn378tlnn9G+fXsA2rdvz+effw7A+vXrOfzww8v27dixI+vXr9/teJs3b2bx4sUMGjSo/k6iFqo09mFmveI2GxHcUR9UJxHtw8wsAzgeGAg8RvAb50m5e6GZ5QGH7YWuHwFOB24wsy4EPzRTVJMDmdl4YDxA69ZtuD67eC+EV7/aNQ2G/lJRqsaeqnGDYm8IBQUFxGIxAGbNmkVBQQGTJ08mMzOT4uLisjqgbHvdunW8/fbbZXUbNmxg5cqVtG7dGgiS/aRJkxg6dChr165l7dq1dR57bVV1cmJm3HoxsAYYtVci2L+cATzh7u+a2abwg8+mZA3N7BDgKOC5vdDvVuBjMzuaIFHfR/CO73j/MbOScP0ed7812YHcfR4wD4I56FSc30rVeTlI3dhTNW5Q7A1h/qnN95jHXbp0KV9++SWHHXYY3bp1o3379mzYsIEOHTqQm5vLSy+9BFC237Rp0zj55JPp168fAOeffz59+/Zlzpy6ncGrbA66Oqo6B32Buw8Ml8HuPh7YuVci2L+MBu4N1+8NtxOdYGbLgU+Bv7v7p3up73sJhrnPABYlqY8f4k6anEVE6svGjRvZvHkzAIWFhTz99NNkZmZy2mmncc89wZeI7rnnHk4//XQATjvtNO6991527NjBmjVreO+99+jTpw8A1113HVu2bGHWrFkNcSo1VtWPVg8CvZKUHbt3w9l3mdmhBPP2R5uZA2kET8b/MaHp8+4+3My+Bywxs0Xh6z1razFwC/C6u281s1ofsGnjNFZPH1br49S3WCxG/pjchg6jRlI19lSNGxR7Q4jFYmzYsIFzzz2XkpISdu3axahRoxg+fDj9+vVj1KhR3HnnnRxxxBE88MADAGRlZTFq1Ci+//3vk56ezu23305aWhrr1q3jxhtvJDMzk169gjR28cUXc+GFFzbkKVZJhQk6fNI4C2hhZj+JqzoYOLAuA9sHnQn8xd0vKi0ws2eBjskah8Pg04CrSX6nXS3hnPbVwLu1PZaISF3r0aMHy5Yt26P80EMP5Zlnnkm6z7XXXsu11+7+yoiOHTtSla8TR1Fld9DdgOFAS+DHceVfA7+oo5j2VaOB6QllDwGTKthnLjDRzDq7+5raBuDu91ZQHT8Hvdzdf17b/kREpOYqTNDu/ijwqJn1c/eX6immfZK75yYpm0PwnfLS7RgQi9supApPcbt7p4Tt+cD8cH1qZfEk7i8iIg2vqnPQy8zsVwTD3WVD2+5+fp1EJSIisp+r6lPcC4DvAKcApfOmX9dVULK7hF8NK11ub+i4RESk7lT1Drqru//UzE5393vM7G/Ak3UZmHzL3e8G7m7oOEREpP5U9Q669FenNoc/dtEC6FQnEYmIiEiV76Dnhb9sNZngJyozgOvrLCoREZH9XJUStLv/X7j6LKBXTIqIiNSxKg1xm1k7M7vTzP4Zbn/fzC6o29BERET2X1Wdg55P8FBYh3D7XeA3dRCPiIiIUPUE3drd7wd2Abh7MVBS8S4iIiJSU1VN0NvClz04gJkdR/BOYREREakDVX2K+3KCp7ePNLMXgDYEL38QERGROlDZ26yOcPe17v6GmQ0geHmGAavdvaiifUVERKTmKhvifiRu/T53X+nubyk5i4iI1K3KErTFrev7zyIiIvWksgTt5ayLiIhIHarsIbGeZraV4E66abhOuO3ufnCdRiciIrKfqvAO2t3T3P1gdz/I3dPD9dJtJWcREanQxx9/zMCBA+nevTtZWVnMnj0bgLy8PI477jhycnLo3bs3r776KgD5+fmccsop5OTkkJOTw4QJE8qOtXTpUrKzs+natSuXXHIJ7vv2wG5Vv2YlIiJSbenp6cycOZNevXrx9ddfc+yxxzJ48GCuuuoqpkyZwpAhQ3j88ce56qqriMViAHTo0IG8vLw9jvXLX/6SefPmcdxxxzF06FCeeOIJhgwZUr8nVI/qLEGbWQmwAmgMFAP3ALPcfZeZ5QKPAmvidpno7k+b2bXAzwh+qWwXcBFwDdCZ4C1abeL2+y/gpnDf180sH1jq7iPDGM4Ehrv7uLi4HgXauns/MzsFuDms6gqsBwqB5cBd4XGHh/udAfw30ITg9ZuT3f2RsG4+MBjo4u47zKw18Lq7d6rkGl0GTAPaufuWsCy3tF8zGwfcEsZ1IPAnd781bDcV+AWwkeDvcZK7PxbWjSf47jrAVuByd18S1sWA9sB2YGd4jF8Ax4fn1hlYHe57g7s/WF78hUUldLrmHxWdYiRdkV3MuBSMG1I39lSNGxR7beRPH0b79u1p3749AAcddBDdu3dn/fr1mBlbtwazplu2bKFDhw4VHYoNGzawdetW+vXrB8DPf/5zHnnkESXoGip09xwAM2sL/I3gPdJTwvrnS5NfKTPrBwwHesUluibuPiKszyUuaYZlif32NrMsd1+ZWGFmLYFeQIGZdXb3Jwl+Y7w0cU1099fj+irdrycwAxjs7mvMrDPwlJl96O7Lw2YlwPnAHVW8PgCjgdeAEQS/d57Mfe5+cfhLbqvN7EF3/zisu9XdZ5hZd+D58DoPJfhQ09/dvzCzXsAjZtbH3T8N9xsTfqA5D7jF3QeH59kJ+Hvp35uIyN6Un5/PsmXL6Nu3L7NmzeKUU05h4sSJ7Nq1ixdffLGs3aeffsoxxxzDwQcfzA033MAJJ5zA+vXr6dixY1mbjh07sn79+oY4jXpTL0Pc7v55eFf3WnjnV572wBfuviPc74sadDcDmASMSVI3ElgMfAacTXD3WhUTgZvcfU0Y1xozmwZcCYwN28wCLjOzP1flgGZ2JMGIwJVhvPMrau/uX5rZ+wTX6OOEurfNrBhoDVwNXFl67cIfmbkH+BXB+7zjvRT2X2Xh3+N4gNat23B9dnF1do+Edk2DO4tUlKqxp2rcoNhro3TIGqCwsJBLL72UCy+8kDfeeIM5c+ZwwQUXMGDAAP7zn//wk5/8hJkzZ7Jz507uvPNOOnTowOrVqxk5ciR33303H3/8MV999VXZMZcvX86mTZt26yMKCgoK9lpM9TYH7e4fmlkjoG1YdIKZ5cU1GQn8C7jezN4Fnia4e3y2ml3dD/yXmXVNUjca+B1Bgn6QqifoLILEH+91gqRXai2whCBhL67CMUcDC4HngW5m1tbdPy+vsZkdQTDMvTxJXV+C6YCNYaxLk8R6bpLDnsruP0ZTKXefB8wDOKJLV5+5IvUeY7giu5hUjBtSN/ZUjRsUe23kj8kFoKioiOHDhzNhwgQuvzyYfTv99NN56KGHMDMGDBjArbfeSm5u0D4Wi5Gbm0tubi4LFy6kXbt29OrVi1mzZpW12bBhA9nZ2WXbUVEa+95Q1Zdl7C3x49HPu3tO3PKBuxcAxxLcoW0E7gvnYaujhGDe9re7dWzWjmCeeYm7vwsUm9nR1Yg78XHBZGU3EdyRVuW6ng3c6+67gIeBn5bT7iwzWwl8CMx29+1xdZeFH3JmAGd5+Y80Jsb6VzNbR3C3/b9ViFVEpEbcnQsuuIDu3buXJWcIHgR79tng/uvf//43Rx11FAAbN26kpCR4WeKHH37Ie++9R5cuXWjfvj0HHXQQL7/8Mu7OX/7yF04//fT6P6F6VG8frcysC0Hy/BzoXl47dy8BYkDMzFYQ3PnNr2Z3CwgSdPw89FnAIcCacN76YIIkeV0VjrcS6M3ud6+9gFUJsb8fJsxRFR3MzHoARxHMY0PwcNaHwO1JmpfOQfcD/mFm/4ybS77V3RPv7FcRfMj5dwWxjgHeBKaHff6konjL07RxGqunD6vJrg0qFouVfbJPNakae6rGDYq9tl544QUWLFhAdnY2OTk5ANx00038+c9/5tJLL6W4uJgDDzyQefPmAfDcc89xxRVX0KJFC9LS0pg7dy6tWrUC4I477mDcuHEUFhYyZMiQffoBMainBG1mbYC5wG3u7kke7Cpt1w3Y5e7vhUU5wEfV7c/di8zsVoKnv0sT1WjgVHd/KeyrM/AUVUvQM4AHzOzf7p4fPkw1ieRv9LoRqOyxydHAVHcvG2I3szVm9t0KzuklM1sAXErC6ECC3wM3m9mp4bx1DjAO6JtwvCIzuw74wMy6u/vblcQsIlJt/fv3L/f7ykuXJs7GwciRIzn00EOTDhP37t2bt956a2+HGFl1maCbhneTpV+zWgD8Ia4+cQ76BoKvT/1v+LR1MfA+4QNJNXAnYfINE+oRwMulleGDXlvNrK+7v1LRgdw9z8yuBhabWWOCr1ld5e55SdquNLM3CO5ay3M2kPjRb1FYXlEsNwNvmNlNFcT6mJkdBrxoZg58DZzj7huStC00s5kED8FdUEG/IiJSz+osQbt7WgV1MYKvXCXzw0r2iyWU5catd4pb3wHEf7HusCTH6xW3nptQt1tf7v4wwVxxsrjGJWxXOGTs7p2TlF0etxkLy+YTN7zv7p8A3wk3p1Zw/Dso5+teSc5zZtx6PlDVeXkREalD9f2QmIiIiFRBan53IEWYWTbB0H68He7eN1l7ERGRUkrQdcjdVxA86CYiIlItGuIWERGJICVoERGRCFKCFhERiSAlaBERkQhSghYREYkgJWgREZEIUoIWERGJICVoERGRCFKCFhERiSAlaBERkQhSghYREYkgJWgREZEIUoIWERGJICVoEZH9yMcff8zAgQPp3r07WVlZzJ49G4CpU6dy2GGHkZOTQ05ODo8//jgA+fn5NG3atKx8woQJZcc69dRT6dmzJ1lZWUyYMIGSkpIGOad9lV43WYfMrARYATQGioF7gFnuvsvMcoGJ7j7czNoBdwKHh23zgav59l3SRwBbwuULdz/JzI4B3gBOdfcn4/p04A/ufkW4PRHIcPep4fbPgasAC5e73H2Gmc0HBoR9AHzj7j/c29dERBpWeno6M2fOpFevXnz99dcce+yxDB48GIDLLruMiRMn7rHPkUceSV5e3h7l999/PwcffDDuzplnnskDDzzA2WefXdensN9Qgq5bhe6eA2BmbYG/AS2AKQnt/ht4yt1nh217xL9LOkyef3f3B+P2GQ0sCf98Mq58B/ATM5vm7l/Ed2JmQ4DfACe7+ydmdiAwNq7JlQl9VHxyRSV0uuYfVW0eGVdkFzMuBeOG1I09VeOGfSv2/OnDaN++Pe3btwfgoIMOonv37qxfv75Gxz/44IMBKC4uZufOnZhZ7YOWMhririfu/jkwHrjY9vyvuD2wLq7t8oqOFe5/JjAOODlMtKWKgXnAZUl2/S3BXfsnYT/b3f3P1TwVEdlH5Ofns2zZMvr27QvAbbfdRo8ePTj//PP56quvytqtWbOGY445hgEDBvD888/vdoxTTjmFtm3bctBBB3HmmWfWa/z7OnP3ho5hn2VmBe6ekVD2FZAJdOfbIe5TgPuAZcDTwN2lSTTcZz5xd9Bm1h/4nbsPMrO/AQ+6+8OlfQIdgOVAT+AXhEPcZrYJ6OzupcPYJPQRP8S90t3HJGk3nuCDBq1btzn2+lmpl9/bNYXPChs6ippJ1dhTNW7Yt2LPPqxF2XphYSGXXnop55xzDieeeCKbNm2iRYsWmBl33XUXX375JVdffTU7d+6ksLCQFi1asHr1aiZPnszdd99N8+bNy461c+dObrjhBk477TR69+5d67gLCgrIyMiovGEEVRb7wIEDl7p7lS6Shrjr3x5jQO7+pJl1AU4FhgDLzOxod99YzjFGA/eG6/cSDFM/HHe8rWb2F+ASoDr/tFQ6xO3u8wju0DmiS1efuSL1/hO6IruYVIwbUjf2VI0b9q3Y88fkAlBUVMTw4cOZMGECl19++R77denSheHDh5Obm7tbeW5uLgsXLqRdu3Z7JOINGzbw2muvJZ3Drq5YLLZH36lib8auIe56FCbhEuDzxDp33+Tuf3P3scBrwInlHCMNGAlcb2b5wP8CQ8zsoISms4ALgOZxZSuBY2t5GiKSwtydCy64gO7du++WnDds2FC2vmjRIo4++mgANm7cWPZ09ocffsh7771Hly5dKCgoKNunuLiYxx9/nMzMzHo8k31fan4sTEFm1gaYC9zm7h4/DW1mPwJedvdvwkR7JLC2nEOdBLzp7qfE7X8PcAbfPvWNu28ys/sJkvRdYfE04PdmNtzdPzWzA4CL3H1OTc6paeM0Vk8fVpNdG1QsFiu7k0g1qRp7qsYN+17sL7zwAgsWLCA7O5ucnBwAbrrpJhYuXEheXh5mRqdOnfjTn/4EwHPPPcf1119Peno6aWlpzJ07l1atWvHZZ59x2mmnsWPHDkpKSvjRj36021ewpPaUoOtWUzPL49uvWS0A/pCk3bHAbWZWTDCq8X/u/lo5xxwNLEooewj4JXEJOjQTuLh0w90fD7/S9XT4oJnzbfIGuMXMrovb7uPuOys4PxFJMf379yfZs0dDhw5N2n7kyJGMHDlyj/J27drx2mvl/TMle4MSdB1y97QK6mJALFy/Bbilgrbjkq3HlT0GPBauZ8SVfwY0S2h7N3B3RX2IiEjD0xy0iIhIBClBi4iIRJAStIiISAQpQYuIiESQErSIiEgEKUGLiIhEkBK0iIhIBClBi4iIRJAStIiISAQpQYuIiESQErSIiEgEKUGLiIhEkBK0iIhIBClBi4iIRJAStIiISAQpQYuIiESQErSIiEgEKUGLSIM5//zzadu2LUcffXRZ2dSpUznssMPIyckhJyeHxx9/HID8/HyaNm1aVj5hwgQAvvnmG4YNG0ZmZiZZWVlcc801DXIuInubEnSEmNkIM3Mzywy3O5nZW+F6rpltMbNlZvaOmc2I22+cmd2W5Hj5ZtY6XHczmxlXN9HMpobrU81svZnlxS0t6/ZsRWDcuHE88cQTe5Rfdtll5OXlkZeXx9ChQ8vKjzzyyLLyuXPnlpVPnDiRd955h2XLlvHCCy/wz3/+s17iF6lL6Q0dgOxmNLAEOBuYmqT+eXcfbmZNgWVmtsjdX6jisXcAPzGzae7+RZL6W919RpLychUWldDpmn9UZ5dIuCK7mHEpGDekbuzJ4s6fPowTTzyR/Pz8Wh27WbNmDBw4EIAmTZrQq1cv1q1bV6tjikSB7qAjwswygOOBCwgSdLncvRDIAw6rRhfFwDzgshqGKFJvbrvtNnr06MH555/PV199VVa+Zs0ajjnmGAYMGMDzzz+/x36bN29m8eLFDBo0qD7DFakTStDRcQbwhLu/C2wys17lNTSzQ4CjgOeq2cftwBgza5Gk7rK44e3/VPO4InvNL3/5Sz744APy8vJo3749V1xxBQDt27dn7dq1LFu2jD/84Q/87Gc/Y+vWrWX7FRcXM3r0aC655BK6dOnSUOGL7DUa4o6O0cCscP3ecPv2hDYnmNlyoBsw3d0/rU4H7r7VzP4CXAIUJlRXaYjbzMYD4wFat27D9dnF1QkhEto1DYZcU1Gqxp4s7lgsBsCnn37Ktm3byrbjZWdn87e//S1p3aGHHsrChQvp1q0bADfffHPZQ2TJ2tdUQUHBXj1efUrV2FM1bti7sStBR4CZHQr8CDjazBxIAxz4Y0LT0jno7wFLwjnovGp2Nwt4A7i7JrG6+zyCoXKO6NLVZ65Ivf+ErsguJhXjhtSNPVnc+WNygz/z82nevDm5ucH2hg0baN++PQC33norffv2JTc3l40bN9KqVSvS0tL48MMP2bhxIz/96U9p1aoV1113Hc2aNeOBBx6gUaO9OzAYi8XKYks1qRp7qsYNezf21Ps/fd90JvAXd7+otMDMngU6Jmvs7u+a2TTgaoI77Spz901mdj/BXPddNQ8ZmjZOY/X0YbU5RIOIxWJlySHVpGrs5cU9evRoYrEYX3zxBR07duR3v/sdsViMvLw8zIxOnTrxpz/9CYDnnnuO66+/nvT0dNLS0pg7dy6tWrVi3bp13HjjjWRmZtKrVzAzdPHFF3PhhRfW5ymK7HVK0NEwGpieUPYQMKmCfeYCE82sc7g9zszOiKs/roJ9ZwIXJ5RdZmbnxG2f4e75FRxDpNYWLly4R9kFF1yQtO3IkSMZOXLkHuUdO3bE3fd6bCINTQk6Atw9N0nZHGBO3HYMiMVtF/LtU9xrgPlJDt0prn1G3PpnQLO47akk/1qXiIg0ED3FLSIiEkFK0CIiIhGkBC0iIhJBStAiIiIRpAQtIiISQUrQIiIiEaQELSIiEkFK0CIiIhGkBC0iIhJBStAiIiIRpAQtIiISQUrQIiIiEaQELSIiEkFK0CIiIhGkBC0iIhJBStAiIiIRpAQtIiISQUrQIiIiEaQELSIiEkFK0CIiIhGkBC0iIhJBStAiIiIRZO7e0DFIijKzr4HVDR1HDbQGvmjoIGooVWNP1bhBsTeEVI0bKo/9u+7epioHSt878ch+arW7927oIKrLzF5PxbghdWNP1bhBsTeEVI0b9m7sGuIWERGJICVoERGRCFKCltqY19AB1FCqxg2pG3uqxg2KvSGkatywF2PXQ2IiIiIRpDtoERGRCFKCFhERiSAlaKk2MzvVzFab2ftmdk1Dx5PIzPLNbIWZ5ZnZ62FZKzN7yszeC/88JK79b8NzWW1mp9RzrHeZ2edm9lZcWbVjNbNjw3N+38zmmJk1UOxTzWx9eO3zzGxo1GI3s8PN7D9m9raZrTSzS8PyyF/3CmKP9HU3swPN7FUzezOM+3dheSpc8/Jir/tr7u5atFR5AdKAD4AuQBPgTeD7DR1XQoz5QOuEst8D14Tr1wA3h+vfD8/hAKBzeG5p9RjriUAv4K3axAq8CvQDDPgnMKSBYp8KTEzSNjKxA+2BXuH6QcC7YXyRv+4VxB7p6x72kRGuNwZeAY5LkWteXux1fs11By3V1Qd4390/dPedwL3A6Q0cU1WcDtwTrt8DnBFXfq+773D3NcD7BOdYL9z9OWBTQnG1YjWz9sDB7v6SB/8K/CVun/qOvTyRid3dN7j7G+H618DbwGGkwHWvIPbyRCJ2DxSEm43DxUmNa15e7OXZa7ErQUt1HQZ8HLe9jor/gWgIDvzLzJaa2fiwrJ27b4DgHzmgbVgexfOpbqyHheuJ5Q3lYjNbHg6Blw5ZRjJ2M+sEHENwV5RS1z0hdoj4dTezNDPLAz4HnnL3lLnm5cQOdXzNlaClupLNmUTtu3rHu3svYAjwKzM7sYK2qXA+pcqLNUrncAdwJJADbABmhuWRi93MMoCHgN+4+9aKmiYpi1rskb/u7l7i7jlAR4I7yqMraB6ZuKHc2Ov8mitBS3WtAw6P2+4IfNJAsSTl7p+Ef34OLCIYsv4sHGIi/PPzsHkUz6e6sa4L1xPL6527fxb+Y7YL+DPfThdEKnYza0yQ4P7q7g+HxSlx3ZPFnirXPYx1MxADTiVFrnmp+Njr45orQUt1vQYcZWadzawJcDbwWAPHVMbMmpvZQaXrwMnAWwQxnhs2Oxd4NFx/DDjbzA4ws87AUQQPcjSkasUaDg1+bWbHhU+F/jxun3pV+o9taATBtYcIxR72cyfwtrv/Ia4q8te9vNijft3NrI2ZtQzXmwInAe+QGtc8aez1cs3r8uk3LfvmAgwleHr0A+Daho4nIbYuBE9QvgmsLI0POBR4Bngv/LNV3D7Xhueymnp4+jkh3oUEw2NFBJ+wL6hJrEDv8B+ID4DbCH8lsAFiXwCsAJaH/1C1j1rsQH+CocXlQF64DE2F615B7JG+7kAPYFkY31vA9WF5Klzz8mKv82uun/oUERGJIA1xi4iIRJAStIiISAQpQYuIiESQErSIiEgEKUGLiIhEkBK0iOzBzEri3tKTF/6sZHWPcYaZfb8OwsPMOpjZg3Vx7Ar6zIl/Y5FIXUtv6ABEJJIKPfhpw9o4A/g7sKqqO5hZursXV9bOg1+LO7PmoVWPmaUT/KRjb+Dx+upX9m+6gxaRKgnfZfts+BKSJ+N+ovEXZvaaBe/LfcjMmpnZD4HTgFvCO/AjzSxmZr3DfVqbWX64Ps7MHjCzxQQvOWkevnzgNTNbZmZ7vC3NzDpZ+B7qcP9HzGyxma0xs4vN7PJw35fNrFXYLmZms8zsRTN7y8z6hOWtwv2Xh+17hOVTzWyemf2L4M1D/w2cFZ7PWWbWJzzWsvDPbnHxPGxmT1jwnuPfx8V9qpm9EV6rZ8KySs9X9lN1+QssWrRoSc0FKOHbX6paRPCKvReBNmH9WcBd4fqhcfvdAPw6XJ8PnBlXFwN6h+utgfxwfRzBL5G1CrdvAs4J11sS/Gpd84T4OhG+hzrc/32C9yO3AbYAE8K6WwleKFHa/5/D9RPj9v9fYEq4/iMgL1yfCiwFmsb1c1tcDAcD6eH6ScBDce0+BFoABwIfEfw2cxuCtxx1DttV+Xy17J+LhrhFJJndhrgteHvP0cBTwc8Ik0bwM58AR5vZDQTJJQN4sgb9PeXupe+WPhk4zcwmhtsHAkcQvPu4PP/x4P3IX5vZFmBxWL6C4KcaSy2E4F3WZnZw+BvL/YGRYfm/zexQM2sRtn/M3QvL6bMFcI+ZHUXw85uN4+qecfctAGa2CvgucAjwnAfvCKaW5yv7ASVoEakKA1a6e78kdfOBM9z9TTMbB+SWc4xivp1WOzChbltCXyPdfXU14tsRt74rbnsXu/87l/jbxpW9BnBbkrpS/0PwwWBE+BBdrJx4SsIYLEn/ULPzlf2A5qBFpCpWA23MrB8Erzw0s6yw7iBggwWvQRwTt8/XYV2pfODYcL2iB7yeBH4dvvEHMzum9uGXOSs8Zn9gS3iX+xxh3GaWC3zhyd8PnXg+LYD14fq4KvT9EjAgfMMRpXPj1O35SgpTghaRSrn7ToKkerOZvUkwN/3DsHoy8ArwFMErBEvdC1wZPvh0JDAD+KWZvUgwB12e/yEYLl4ePgj2P3vxVL4K+59L8PYtCOaae5vZcmA6377+MNF/gO+XPiQG/B6YZmYvEAz5V8jdNwLjgYfDa3hfWFWX5yspTG+zEpH9gpnFgInu/npDxyJSFbqDFhERiSDdQYuIiESQ7qBFREQiSAlaREQkgpSgRUREIkgJWkREJIKUoEVERCLo/wOsA9/eTKyHeAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 读取数据\n",
    "data = pd.read_csv(\"https://cdn.coggle.club/kaggle-flight-delays/flights_10k.csv.zip\")\n",
    "\n",
    "# 提取有用的列\n",
    "data = data[[\"MONTH\",\"DAY\",\"DAY_OF_WEEK\",\"AIRLINE\",\"FLIGHT_NUMBER\",\"DESTINATION_AIRPORT\",\n",
    "                 \"ORIGIN_AIRPORT\",\"AIR_TIME\", \"DEPARTURE_TIME\",\"DISTANCE\",\"ARRIVAL_DELAY\"]]\n",
    "data.dropna(inplace=True)\n",
    "\n",
    "# 筛选出部分数据\n",
    "data[\"ARRIVAL_DELAY\"] = (data[\"ARRIVAL_DELAY\"]>10)*1\n",
    "\n",
    "# 进行编码\n",
    "cols = [\"AIRLINE\",\"FLIGHT_NUMBER\",\"DESTINATION_AIRPORT\",\"ORIGIN_AIRPORT\"]\n",
    "for item in cols:\n",
    "    data[item] = data[item].astype(\"category\").cat.codes +1\n",
    "\n",
    "# 划分训练集和测试集\n",
    "train, test, y_train, y_test = train_test_split(data.drop([\"ARRIVAL_DELAY\"], axis=1), data[\"ARRIVAL_DELAY\"], random_state=10, test_size=0.25)\n",
    "\n",
    "\n",
    "dtrain = lgb.Dataset(train, y_train, free_raw_data=False, silent=True)\n",
    "lgb_params = {\n",
    "        'learning_rate': 0.01,\n",
    "        'max_depth': 5,\n",
    "        'n_estimators': 1000,\n",
    "        'num_leaves': 128,\n",
    "        'subsample_for_bin': 1000,\n",
    "        'refit_decay_rate': 0.1,\n",
    "        'verbose': -1\n",
    "    \n",
    "    }\n",
    "\n",
    "clf = lgb.train(params=lgb_params, train_set=dtrain, num_boost_round=200)\n",
    "\n",
    "\n",
    "lgb.plot_importance(clf, max_num_features=15)\n",
    "plt.title('Important Features')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "id": "74d14aab",
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "    <style>\n",
       "    table.eli5-weights tr:hover {\n",
       "        filter: brightness(85%);\n",
       "    }\n",
       "</style>\n",
       "\n",
       "\n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "        <table class=\"eli5-weights eli5-feature-importances\" style=\"border-collapse: collapse; border: none; margin-top: 0em; table-layout: auto;\">\n",
       "    <thead>\n",
       "    <tr style=\"border: none;\">\n",
       "        <th style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">Weight</th>\n",
       "        <th style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">Feature</th>\n",
       "    </tr>\n",
       "    </thead>\n",
       "    <tbody>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 80.00%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0409\n",
       "                \n",
       "                    &plusmn; 0.0044\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                DEPARTURE_TIME\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 86.89%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0224\n",
       "                \n",
       "                    &plusmn; 0.0068\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                AIR_TIME\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 89.21%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0169\n",
       "                \n",
       "                    &plusmn; 0.0046\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                DESTINATION_AIRPORT\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 89.70%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0158\n",
       "                \n",
       "                    &plusmn; 0.0077\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                DISTANCE\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 90.23%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0147\n",
       "                \n",
       "                    &plusmn; 0.0054\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                ORIGIN_AIRPORT\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 95.08%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0055\n",
       "                \n",
       "                    &plusmn; 0.0032\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                FLIGHT_NUMBER\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 97.72%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0018\n",
       "                \n",
       "                    &plusmn; 0.0039\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                AIRLINE\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(0, 100.00%, 100.00%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0\n",
       "                \n",
       "                    &plusmn; 0.0000\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                DAY_OF_WEEK\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(0, 100.00%, 100.00%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0\n",
       "                \n",
       "                    &plusmn; 0.0000\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                DAY\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(0, 100.00%, 100.00%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0\n",
       "                \n",
       "                    &plusmn; 0.0000\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                MONTH\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "    \n",
       "    </tbody>\n",
       "</table>\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "execution_count": 148,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 排列特征重要性\n",
    "import eli5\n",
    "from eli5.sklearn import PermutationImportance\n",
    "\n",
    "perm = PermutationImportance(model, random_state=1).fit(test, y_test)\n",
    "eli5.show_weights(perm, feature_names = test.columns.tolist())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "88b27607",
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████| 80/80 [00:21<00:00,  3.79it/s]\n"
     ]
    }
   ],
   "source": [
    "# null importance\n",
    "\n",
    "def get_feature_importance(data, shuffle=False):\n",
    "    \n",
    "    X = data.drop([\"ARRIVAL_DELAY\"], axis=1)\n",
    "    y = data[\"ARRIVAL_DELAY\"].copy()\n",
    "    if shuffle == True:\n",
    "        y = data['ARRIVAL_DELAY'].copy().sample(frac=1.0)\n",
    "    \n",
    "    # 划分训练集和测试集\n",
    "    train, test, y_train, y_test = train_test_split(X, y, random_state=10, test_size=0.25)\n",
    "\n",
    "\n",
    "    dtrain = lgb.Dataset(train, y_train, free_raw_data=False, silent=True)\n",
    "    lgb_params = {\n",
    "            'learning_rate': 0.01,\n",
    "            'max_depth': 5,\n",
    "            'n_estimators': 1000,\n",
    "            'num_leaves': 128,\n",
    "            'subsample_for_bin': 1000,\n",
    "            'refit_decay_rate': 0.1,\n",
    "             'verbose': -1\n",
    "        }\n",
    "\n",
    "    clf = lgb.train(params=lgb_params, train_set=dtrain, num_boost_round=200)\n",
    "    \n",
    "\n",
    "    # 获取特征重要性\n",
    "    imp_df = pd.DataFrame()\n",
    "    imp_df[\"feature\"] = list(train.columns)\n",
    "    imp_df[\"importance_gain\"] = clf.feature_importance(importance_type='gain')\n",
    "    imp_df[\"importance_split\"] = clf.feature_importance(importance_type='split')\n",
    "    imp_df['trn_score'] = roc_auc_score(y_test, clf.predict(test))\n",
    "    return imp_df\n",
    "# 正特征重要性\n",
    "actual_imp_df = get_feature_importance(data)\n",
    "# 负特征重要性\n",
    "null_imp_df = pd.DataFrame()\n",
    "nb_runs = 80\n",
    "import time\n",
    "start = time.time()\n",
    "dsp = ''\n",
    "for i in tqdm(range(nb_runs)):\n",
    "    # Get current run importances\n",
    "    imp_df = get_feature_importance(data=data, shuffle=True)\n",
    "    imp_df['run'] = i + 1 \n",
    "    # Concat the latest importances with the old ones\n",
    "    null_imp_df = pd.concat([null_imp_df, imp_df], axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "ee85f878",
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>feature</th>\n",
       "      <th>importance_gain</th>\n",
       "      <th>importance_split</th>\n",
       "      <th>trn_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>MONTH</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0</td>\n",
       "      <td>0.762327</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>DAY</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0</td>\n",
       "      <td>0.762327</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>DAY_OF_WEEK</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0</td>\n",
       "      <td>0.762327</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>FLIGHT_NUMBER</td>\n",
       "      <td>1861.836857</td>\n",
       "      <td>3132</td>\n",
       "      <td>0.762327</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>AIRLINE</td>\n",
       "      <td>1902.242592</td>\n",
       "      <td>1552</td>\n",
       "      <td>0.762327</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>DISTANCE</td>\n",
       "      <td>1931.296176</td>\n",
       "      <td>2553</td>\n",
       "      <td>0.762327</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>AIR_TIME</td>\n",
       "      <td>2056.819326</td>\n",
       "      <td>3002</td>\n",
       "      <td>0.762327</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>ORIGIN_AIRPORT</td>\n",
       "      <td>2714.062791</td>\n",
       "      <td>3138</td>\n",
       "      <td>0.762327</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>DESTINATION_AIRPORT</td>\n",
       "      <td>3916.088581</td>\n",
       "      <td>2850</td>\n",
       "      <td>0.762327</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>DEPARTURE_TIME</td>\n",
       "      <td>5772.896138</td>\n",
       "      <td>3266</td>\n",
       "      <td>0.762327</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               feature  importance_gain  importance_split  trn_score\n",
       "0                MONTH         0.000000                 0   0.762327\n",
       "1                  DAY         0.000000                 0   0.762327\n",
       "2          DAY_OF_WEEK         0.000000                 0   0.762327\n",
       "4        FLIGHT_NUMBER      1861.836857              3132   0.762327\n",
       "3              AIRLINE      1902.242592              1552   0.762327\n",
       "9             DISTANCE      1931.296176              2553   0.762327\n",
       "7             AIR_TIME      2056.819326              3002   0.762327\n",
       "6       ORIGIN_AIRPORT      2714.062791              3138   0.762327\n",
       "5  DESTINATION_AIRPORT      3916.088581              2850   0.762327\n",
       "8       DEPARTURE_TIME      5772.896138              3266   0.762327"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "actual_imp_df.sort_values('importance_gain')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "db837a2e",
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>feature</th>\n",
       "      <th>importance_gain</th>\n",
       "      <th>importance_split</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>MONTH</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>DAY</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>DAY_OF_WEEK</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>AIRLINE</td>\n",
       "      <td>480.783022</td>\n",
       "      <td>901.0875</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>DESTINATION_AIRPORT</td>\n",
       "      <td>1152.733965</td>\n",
       "      <td>2241.4000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>ORIGIN_AIRPORT</td>\n",
       "      <td>1191.787954</td>\n",
       "      <td>2333.3250</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>DISTANCE</td>\n",
       "      <td>1305.996216</td>\n",
       "      <td>2422.4000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>AIR_TIME</td>\n",
       "      <td>1465.937798</td>\n",
       "      <td>2781.6500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>FLIGHT_NUMBER</td>\n",
       "      <td>1755.221838</td>\n",
       "      <td>3290.2375</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>DEPARTURE_TIME</td>\n",
       "      <td>1960.960761</td>\n",
       "      <td>3714.4000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               feature  importance_gain  importance_split\n",
       "0                MONTH         0.000000            0.0000\n",
       "5                  DAY         0.000000            0.0000\n",
       "9          DAY_OF_WEEK         0.000000            0.0000\n",
       "4              AIRLINE       480.783022          901.0875\n",
       "7  DESTINATION_AIRPORT      1152.733965         2241.4000\n",
       "6       ORIGIN_AIRPORT      1191.787954         2333.3250\n",
       "1             DISTANCE      1305.996216         2422.4000\n",
       "2             AIR_TIME      1465.937798         2781.6500\n",
       "8        FLIGHT_NUMBER      1755.221838         3290.2375\n",
       "3       DEPARTURE_TIME      1960.960761         3714.4000"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "null_imp_list = []\n",
    "for i in list(set(actual_imp_df['feature'])):\n",
    "    tmp_df = null_imp_df[null_imp_df['feature'] == i]\n",
    "    null_imp_list.append({\n",
    "        'feature': i,\n",
    "        'importance_gain': tmp_df['importance_gain'].mean(),\n",
    "        'importance_split': tmp_df['importance_split'].mean(),\n",
    "    })\n",
    "null_imp_df = pd.DataFrame(null_imp_list)\n",
    "null_imp_df.sort_values('importance_gain')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "142bf8de",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "# 任务8：自定义损失函数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "0724e878",
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1]\tvalid_0's error: 0.209341\n",
      "[2]\tvalid_0's error: 0.209341\n",
      "[3]\tvalid_0's error: 0.209341\n",
      "[4]\tvalid_0's error: 0.209341\n",
      "[5]\tvalid_0's error: 0.209341\n",
      "[6]\tvalid_0's error: 0.209341\n",
      "[7]\tvalid_0's error: 0.209341\n",
      "[8]\tvalid_0's error: 0.209341\n",
      "[9]\tvalid_0's error: 0.209341\n",
      "[10]\tvalid_0's error: 0.209341\n",
      "[11]\tvalid_0's error: 0.209341\n",
      "[12]\tvalid_0's error: 0.209341\n",
      "[13]\tvalid_0's error: 0.209341\n",
      "[14]\tvalid_0's error: 0.209341\n",
      "[15]\tvalid_0's error: 0.209341\n",
      "[16]\tvalid_0's error: 0.209341\n",
      "[17]\tvalid_0's error: 0.209341\n",
      "[18]\tvalid_0's error: 0.209341\n",
      "[19]\tvalid_0's error: 0.209341\n",
      "[20]\tvalid_0's error: 0.209341\n",
      "[21]\tvalid_0's error: 0.209341\n",
      "[22]\tvalid_0's error: 0.209341\n",
      "[23]\tvalid_0's error: 0.209341\n",
      "[24]\tvalid_0's error: 0.209341\n",
      "[25]\tvalid_0's error: 0.209341\n",
      "[26]\tvalid_0's error: 0.209341\n",
      "[27]\tvalid_0's error: 0.209341\n",
      "[28]\tvalid_0's error: 0.209341\n",
      "[29]\tvalid_0's error: 0.209341\n",
      "[30]\tvalid_0's error: 0.209341\n",
      "[31]\tvalid_0's error: 0.209341\n",
      "[32]\tvalid_0's error: 0.209341\n",
      "[33]\tvalid_0's error: 0.209341\n",
      "[34]\tvalid_0's error: 0.209341\n",
      "[35]\tvalid_0's error: 0.209341\n",
      "[36]\tvalid_0's error: 0.209341\n",
      "[37]\tvalid_0's error: 0.209341\n",
      "[38]\tvalid_0's error: 0.209341\n",
      "[39]\tvalid_0's error: 0.209341\n",
      "[40]\tvalid_0's error: 0.209341\n",
      "[41]\tvalid_0's error: 0.209341\n",
      "[42]\tvalid_0's error: 0.209341\n",
      "[43]\tvalid_0's error: 0.209341\n",
      "[44]\tvalid_0's error: 0.209341\n",
      "[45]\tvalid_0's error: 0.209341\n",
      "[46]\tvalid_0's error: 0.209341\n",
      "[47]\tvalid_0's error: 0.209341\n",
      "[48]\tvalid_0's error: 0.209341\n",
      "[49]\tvalid_0's error: 0.209341\n",
      "[50]\tvalid_0's error: 0.209341\n",
      "[51]\tvalid_0's error: 0.209341\n",
      "[52]\tvalid_0's error: 0.209341\n",
      "[53]\tvalid_0's error: 0.209341\n",
      "[54]\tvalid_0's error: 0.209341\n",
      "[55]\tvalid_0's error: 0.209341\n",
      "[56]\tvalid_0's error: 0.209341\n",
      "[57]\tvalid_0's error: 0.209341\n",
      "[58]\tvalid_0's error: 0.209341\n",
      "[59]\tvalid_0's error: 0.209341\n",
      "[60]\tvalid_0's error: 0.209341\n",
      "[61]\tvalid_0's error: 0.209341\n",
      "[62]\tvalid_0's error: 0.209341\n",
      "[63]\tvalid_0's error: 0.209341\n",
      "[64]\tvalid_0's error: 0.209341\n",
      "[65]\tvalid_0's error: 0.209341\n",
      "[66]\tvalid_0's error: 0.209341\n",
      "[67]\tvalid_0's error: 0.209341\n",
      "[68]\tvalid_0's error: 0.209341\n",
      "[69]\tvalid_0's error: 0.209341\n",
      "[70]\tvalid_0's error: 0.209341\n",
      "[71]\tvalid_0's error: 0.209341\n",
      "[72]\tvalid_0's error: 0.209341\n",
      "[73]\tvalid_0's error: 0.209341\n",
      "[74]\tvalid_0's error: 0.209341\n",
      "[75]\tvalid_0's error: 0.209341\n",
      "[76]\tvalid_0's error: 0.209341\n",
      "[77]\tvalid_0's error: 0.209341\n",
      "[78]\tvalid_0's error: 0.209341\n",
      "[79]\tvalid_0's error: 0.209341\n",
      "[80]\tvalid_0's error: 0.209341\n",
      "[81]\tvalid_0's error: 0.209341\n",
      "[82]\tvalid_0's error: 0.209341\n",
      "[83]\tvalid_0's error: 0.209341\n",
      "[84]\tvalid_0's error: 0.209341\n",
      "[85]\tvalid_0's error: 0.209341\n",
      "[86]\tvalid_0's error: 0.209341\n",
      "[87]\tvalid_0's error: 0.209341\n",
      "[88]\tvalid_0's error: 0.209341\n",
      "[89]\tvalid_0's error: 0.208924\n",
      "[90]\tvalid_0's error: 0.207673\n",
      "[91]\tvalid_0's error: 0.207256\n",
      "[92]\tvalid_0's error: 0.204754\n",
      "[93]\tvalid_0's error: 0.204337\n",
      "[94]\tvalid_0's error: 0.20392\n",
      "[95]\tvalid_0's error: 0.203503\n",
      "[96]\tvalid_0's error: 0.203086\n",
      "[97]\tvalid_0's error: 0.203086\n",
      "[98]\tvalid_0's error: 0.203086\n",
      "[99]\tvalid_0's error: 0.203086\n",
      "[100]\tvalid_0's error: 0.202669\n",
      "[101]\tvalid_0's error: 0.201418\n",
      "[102]\tvalid_0's error: 0.19975\n",
      "[103]\tvalid_0's error: 0.198499\n",
      "[104]\tvalid_0's error: 0.198499\n",
      "[105]\tvalid_0's error: 0.198082\n",
      "[106]\tvalid_0's error: 0.198082\n",
      "[107]\tvalid_0's error: 0.198082\n",
      "[108]\tvalid_0's error: 0.197665\n",
      "[109]\tvalid_0's error: 0.196831\n",
      "[110]\tvalid_0's error: 0.196831\n",
      "[111]\tvalid_0's error: 0.196831\n",
      "[112]\tvalid_0's error: 0.196831\n",
      "[113]\tvalid_0's error: 0.196831\n",
      "[114]\tvalid_0's error: 0.196831\n",
      "[115]\tvalid_0's error: 0.196831\n",
      "[116]\tvalid_0's error: 0.196831\n",
      "[117]\tvalid_0's error: 0.196831\n",
      "[118]\tvalid_0's error: 0.196831\n",
      "[119]\tvalid_0's error: 0.196831\n",
      "[120]\tvalid_0's error: 0.196831\n",
      "[121]\tvalid_0's error: 0.196831\n",
      "[122]\tvalid_0's error: 0.196831\n",
      "[123]\tvalid_0's error: 0.196831\n",
      "[124]\tvalid_0's error: 0.196414\n",
      "[125]\tvalid_0's error: 0.196414\n",
      "[126]\tvalid_0's error: 0.196414\n",
      "[127]\tvalid_0's error: 0.196414\n",
      "[128]\tvalid_0's error: 0.196414\n",
      "[129]\tvalid_0's error: 0.196414\n",
      "[130]\tvalid_0's error: 0.195997\n",
      "[131]\tvalid_0's error: 0.195997\n",
      "[132]\tvalid_0's error: 0.196414\n",
      "[133]\tvalid_0's error: 0.196414\n",
      "[134]\tvalid_0's error: 0.196414\n",
      "[135]\tvalid_0's error: 0.196414\n",
      "[136]\tvalid_0's error: 0.196414\n",
      "[137]\tvalid_0's error: 0.196414\n",
      "[138]\tvalid_0's error: 0.196414\n",
      "[139]\tvalid_0's error: 0.196414\n",
      "[140]\tvalid_0's error: 0.196414\n",
      "[141]\tvalid_0's error: 0.196414\n",
      "[142]\tvalid_0's error: 0.196414\n",
      "[143]\tvalid_0's error: 0.196414\n",
      "[144]\tvalid_0's error: 0.196414\n",
      "[145]\tvalid_0's error: 0.19558\n",
      "[146]\tvalid_0's error: 0.19558\n",
      "[147]\tvalid_0's error: 0.19558\n",
      "[148]\tvalid_0's error: 0.19558\n",
      "[149]\tvalid_0's error: 0.19558\n",
      "[150]\tvalid_0's error: 0.19558\n",
      "[151]\tvalid_0's error: 0.19558\n",
      "[152]\tvalid_0's error: 0.19558\n",
      "[153]\tvalid_0's error: 0.19558\n",
      "[154]\tvalid_0's error: 0.19558\n",
      "[155]\tvalid_0's error: 0.19558\n",
      "[156]\tvalid_0's error: 0.19558\n",
      "[157]\tvalid_0's error: 0.195163\n",
      "[158]\tvalid_0's error: 0.195163\n",
      "[159]\tvalid_0's error: 0.195163\n",
      "[160]\tvalid_0's error: 0.195163\n",
      "[161]\tvalid_0's error: 0.194329\n",
      "[162]\tvalid_0's error: 0.194329\n",
      "[163]\tvalid_0's error: 0.194329\n",
      "[164]\tvalid_0's error: 0.194329\n",
      "[165]\tvalid_0's error: 0.194329\n",
      "[166]\tvalid_0's error: 0.194329\n",
      "[167]\tvalid_0's error: 0.194329\n",
      "[168]\tvalid_0's error: 0.194746\n",
      "[169]\tvalid_0's error: 0.194746\n",
      "[170]\tvalid_0's error: 0.194329\n",
      "[171]\tvalid_0's error: 0.194329\n",
      "[172]\tvalid_0's error: 0.194329\n",
      "[173]\tvalid_0's error: 0.194329\n",
      "[174]\tvalid_0's error: 0.194329\n",
      "[175]\tvalid_0's error: 0.194329\n",
      "[176]\tvalid_0's error: 0.194329\n",
      "[177]\tvalid_0's error: 0.194329\n",
      "[178]\tvalid_0's error: 0.194329\n",
      "[179]\tvalid_0's error: 0.194329\n",
      "[180]\tvalid_0's error: 0.194329\n",
      "[181]\tvalid_0's error: 0.194329\n",
      "[182]\tvalid_0's error: 0.194329\n",
      "[183]\tvalid_0's error: 0.194329\n",
      "[184]\tvalid_0's error: 0.194746\n",
      "[185]\tvalid_0's error: 0.194746\n",
      "[186]\tvalid_0's error: 0.194329\n",
      "[187]\tvalid_0's error: 0.194746\n",
      "[188]\tvalid_0's error: 0.194746\n",
      "[189]\tvalid_0's error: 0.194746\n",
      "[190]\tvalid_0's error: 0.194746\n",
      "[191]\tvalid_0's error: 0.194746\n",
      "[192]\tvalid_0's error: 0.194746\n",
      "[193]\tvalid_0's error: 0.194746\n",
      "[194]\tvalid_0's error: 0.194746\n",
      "[195]\tvalid_0's error: 0.194746\n",
      "[196]\tvalid_0's error: 0.194746\n",
      "[197]\tvalid_0's error: 0.194746\n",
      "[198]\tvalid_0's error: 0.194746\n",
      "[199]\tvalid_0's error: 0.194746\n",
      "[200]\tvalid_0's error: 0.194746\n",
      "[201]\tvalid_0's error: 0.194746\n",
      "[202]\tvalid_0's error: 0.194746\n",
      "[203]\tvalid_0's error: 0.194746\n",
      "[204]\tvalid_0's error: 0.194746\n",
      "[205]\tvalid_0's error: 0.194746\n",
      "[206]\tvalid_0's error: 0.194746\n",
      "[207]\tvalid_0's error: 0.194746\n",
      "[208]\tvalid_0's error: 0.194746\n",
      "[209]\tvalid_0's error: 0.194746\n",
      "[210]\tvalid_0's error: 0.194746\n",
      "[211]\tvalid_0's error: 0.194746\n",
      "[212]\tvalid_0's error: 0.194746\n",
      "[213]\tvalid_0's error: 0.194746\n",
      "[214]\tvalid_0's error: 0.194746\n",
      "[215]\tvalid_0's error: 0.194746\n",
      "[216]\tvalid_0's error: 0.194746\n",
      "[217]\tvalid_0's error: 0.194746\n",
      "[218]\tvalid_0's error: 0.194746\n",
      "[219]\tvalid_0's error: 0.194746\n",
      "[220]\tvalid_0's error: 0.194746\n",
      "[221]\tvalid_0's error: 0.194746\n",
      "[222]\tvalid_0's error: 0.194746\n",
      "[223]\tvalid_0's error: 0.194746\n",
      "[224]\tvalid_0's error: 0.194746\n",
      "[225]\tvalid_0's error: 0.194746\n",
      "[226]\tvalid_0's error: 0.194746\n",
      "[227]\tvalid_0's error: 0.194746\n",
      "[228]\tvalid_0's error: 0.194746\n",
      "[229]\tvalid_0's error: 0.194746\n",
      "[230]\tvalid_0's error: 0.194746\n",
      "[231]\tvalid_0's error: 0.194746\n",
      "[232]\tvalid_0's error: 0.194746\n",
      "[233]\tvalid_0's error: 0.194746\n",
      "[234]\tvalid_0's error: 0.194746\n",
      "[235]\tvalid_0's error: 0.194746\n",
      "[236]\tvalid_0's error: 0.194746\n",
      "[237]\tvalid_0's error: 0.194746\n",
      "[238]\tvalid_0's error: 0.194746\n",
      "[239]\tvalid_0's error: 0.194746\n",
      "[240]\tvalid_0's error: 0.194746\n",
      "[241]\tvalid_0's error: 0.194746\n",
      "[242]\tvalid_0's error: 0.194746\n",
      "[243]\tvalid_0's error: 0.194746\n",
      "[244]\tvalid_0's error: 0.194746\n",
      "[245]\tvalid_0's error: 0.194746\n",
      "[246]\tvalid_0's error: 0.194746\n",
      "[247]\tvalid_0's error: 0.194746\n",
      "[248]\tvalid_0's error: 0.194746\n",
      "[249]\tvalid_0's error: 0.194746\n",
      "[250]\tvalid_0's error: 0.194746\n",
      "[251]\tvalid_0's error: 0.194746\n",
      "[252]\tvalid_0's error: 0.194746\n",
      "[253]\tvalid_0's error: 0.194746\n",
      "[254]\tvalid_0's error: 0.194746\n",
      "[255]\tvalid_0's error: 0.194746\n",
      "[256]\tvalid_0's error: 0.194746\n",
      "[257]\tvalid_0's error: 0.194746\n",
      "[258]\tvalid_0's error: 0.194746\n",
      "[259]\tvalid_0's error: 0.194746\n",
      "[260]\tvalid_0's error: 0.194746\n",
      "[261]\tvalid_0's error: 0.194746\n",
      "[262]\tvalid_0's error: 0.194746\n",
      "[263]\tvalid_0's error: 0.194746\n",
      "[264]\tvalid_0's error: 0.194746\n",
      "[265]\tvalid_0's error: 0.194746\n",
      "[266]\tvalid_0's error: 0.194746\n",
      "[267]\tvalid_0's error: 0.194746\n",
      "[268]\tvalid_0's error: 0.194746\n",
      "[269]\tvalid_0's error: 0.194746\n",
      "[270]\tvalid_0's error: 0.194746\n",
      "[271]\tvalid_0's error: 0.194746\n",
      "[272]\tvalid_0's error: 0.194746\n",
      "[273]\tvalid_0's error: 0.194746\n",
      "[274]\tvalid_0's error: 0.194746\n",
      "[275]\tvalid_0's error: 0.194746\n",
      "[276]\tvalid_0's error: 0.194746\n",
      "[277]\tvalid_0's error: 0.194746\n",
      "[278]\tvalid_0's error: 0.194746\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[279]\tvalid_0's error: 0.194746\n",
      "[280]\tvalid_0's error: 0.194746\n",
      "[281]\tvalid_0's error: 0.194746\n",
      "[282]\tvalid_0's error: 0.194746\n",
      "[283]\tvalid_0's error: 0.194746\n",
      "[284]\tvalid_0's error: 0.194746\n",
      "[285]\tvalid_0's error: 0.194746\n",
      "[286]\tvalid_0's error: 0.194746\n",
      "[287]\tvalid_0's error: 0.194746\n",
      "[288]\tvalid_0's error: 0.194746\n",
      "[289]\tvalid_0's error: 0.194329\n",
      "[290]\tvalid_0's error: 0.194329\n",
      "[291]\tvalid_0's error: 0.194329\n",
      "[292]\tvalid_0's error: 0.194329\n",
      "[293]\tvalid_0's error: 0.194329\n",
      "[294]\tvalid_0's error: 0.194329\n",
      "[295]\tvalid_0's error: 0.194329\n",
      "[296]\tvalid_0's error: 0.194329\n",
      "[297]\tvalid_0's error: 0.193912\n",
      "[298]\tvalid_0's error: 0.193912\n",
      "[299]\tvalid_0's error: 0.193912\n",
      "[300]\tvalid_0's error: 0.193912\n",
      "[301]\tvalid_0's error: 0.193912\n",
      "[302]\tvalid_0's error: 0.193912\n",
      "[303]\tvalid_0's error: 0.193912\n",
      "[304]\tvalid_0's error: 0.193912\n",
      "[305]\tvalid_0's error: 0.193912\n",
      "[306]\tvalid_0's error: 0.193912\n",
      "[307]\tvalid_0's error: 0.193912\n",
      "[308]\tvalid_0's error: 0.193912\n",
      "[309]\tvalid_0's error: 0.193912\n",
      "[310]\tvalid_0's error: 0.193912\n",
      "[311]\tvalid_0's error: 0.193912\n",
      "[312]\tvalid_0's error: 0.193912\n",
      "[313]\tvalid_0's error: 0.193912\n",
      "[314]\tvalid_0's error: 0.193912\n",
      "[315]\tvalid_0's error: 0.193912\n",
      "[316]\tvalid_0's error: 0.193912\n",
      "[317]\tvalid_0's error: 0.193912\n",
      "[318]\tvalid_0's error: 0.193912\n",
      "[319]\tvalid_0's error: 0.193912\n",
      "[320]\tvalid_0's error: 0.193912\n",
      "[321]\tvalid_0's error: 0.193912\n",
      "[322]\tvalid_0's error: 0.193912\n",
      "[323]\tvalid_0's error: 0.193912\n",
      "[324]\tvalid_0's error: 0.193912\n",
      "[325]\tvalid_0's error: 0.193912\n",
      "[326]\tvalid_0's error: 0.194329\n",
      "[327]\tvalid_0's error: 0.194329\n",
      "[328]\tvalid_0's error: 0.194329\n",
      "[329]\tvalid_0's error: 0.194329\n",
      "[330]\tvalid_0's error: 0.194329\n",
      "[331]\tvalid_0's error: 0.194329\n",
      "[332]\tvalid_0's error: 0.194329\n",
      "[333]\tvalid_0's error: 0.194329\n",
      "[334]\tvalid_0's error: 0.194329\n",
      "[335]\tvalid_0's error: 0.194329\n",
      "[336]\tvalid_0's error: 0.194329\n",
      "[337]\tvalid_0's error: 0.194329\n",
      "[338]\tvalid_0's error: 0.194329\n",
      "[339]\tvalid_0's error: 0.194329\n",
      "[340]\tvalid_0's error: 0.194329\n",
      "[341]\tvalid_0's error: 0.194329\n",
      "[342]\tvalid_0's error: 0.194329\n",
      "[343]\tvalid_0's error: 0.194329\n",
      "[344]\tvalid_0's error: 0.194329\n",
      "[345]\tvalid_0's error: 0.194329\n",
      "[346]\tvalid_0's error: 0.194329\n",
      "[347]\tvalid_0's error: 0.194329\n",
      "[348]\tvalid_0's error: 0.194329\n",
      "[349]\tvalid_0's error: 0.194329\n",
      "[350]\tvalid_0's error: 0.194329\n",
      "[351]\tvalid_0's error: 0.194329\n",
      "[352]\tvalid_0's error: 0.194329\n",
      "[353]\tvalid_0's error: 0.194329\n",
      "[354]\tvalid_0's error: 0.194329\n",
      "[355]\tvalid_0's error: 0.194329\n",
      "[356]\tvalid_0's error: 0.194329\n",
      "[357]\tvalid_0's error: 0.194329\n",
      "[358]\tvalid_0's error: 0.194329\n",
      "[359]\tvalid_0's error: 0.194329\n",
      "[360]\tvalid_0's error: 0.194329\n",
      "[361]\tvalid_0's error: 0.194329\n",
      "[362]\tvalid_0's error: 0.194329\n",
      "[363]\tvalid_0's error: 0.194329\n",
      "[364]\tvalid_0's error: 0.194329\n",
      "[365]\tvalid_0's error: 0.194329\n",
      "[366]\tvalid_0's error: 0.194329\n",
      "[367]\tvalid_0's error: 0.194329\n",
      "[368]\tvalid_0's error: 0.194746\n",
      "[369]\tvalid_0's error: 0.194746\n",
      "[370]\tvalid_0's error: 0.194329\n",
      "[371]\tvalid_0's error: 0.194329\n",
      "[372]\tvalid_0's error: 0.194329\n",
      "[373]\tvalid_0's error: 0.194329\n",
      "[374]\tvalid_0's error: 0.194746\n",
      "[375]\tvalid_0's error: 0.194746\n",
      "[376]\tvalid_0's error: 0.194746\n",
      "[377]\tvalid_0's error: 0.194746\n",
      "[378]\tvalid_0's error: 0.194329\n",
      "[379]\tvalid_0's error: 0.194329\n",
      "[380]\tvalid_0's error: 0.194329\n",
      "[381]\tvalid_0's error: 0.194329\n",
      "[382]\tvalid_0's error: 0.194329\n",
      "[383]\tvalid_0's error: 0.194329\n",
      "[384]\tvalid_0's error: 0.194329\n",
      "[385]\tvalid_0's error: 0.194329\n",
      "[386]\tvalid_0's error: 0.194329\n",
      "[387]\tvalid_0's error: 0.194329\n",
      "[388]\tvalid_0's error: 0.194329\n",
      "[389]\tvalid_0's error: 0.194329\n",
      "[390]\tvalid_0's error: 0.194329\n",
      "[391]\tvalid_0's error: 0.194329\n",
      "[392]\tvalid_0's error: 0.194329\n",
      "[393]\tvalid_0's error: 0.194329\n",
      "[394]\tvalid_0's error: 0.194329\n",
      "[395]\tvalid_0's error: 0.194329\n",
      "[396]\tvalid_0's error: 0.194329\n",
      "[397]\tvalid_0's error: 0.194329\n",
      "[398]\tvalid_0's error: 0.194329\n",
      "[399]\tvalid_0's error: 0.194329\n",
      "[400]\tvalid_0's error: 0.194329\n",
      "[401]\tvalid_0's error: 0.194329\n",
      "[402]\tvalid_0's error: 0.194329\n",
      "[403]\tvalid_0's error: 0.194329\n",
      "[404]\tvalid_0's error: 0.194329\n",
      "[405]\tvalid_0's error: 0.194329\n",
      "[406]\tvalid_0's error: 0.194329\n",
      "[407]\tvalid_0's error: 0.194329\n",
      "[408]\tvalid_0's error: 0.194329\n",
      "[409]\tvalid_0's error: 0.194329\n",
      "[410]\tvalid_0's error: 0.194329\n",
      "[411]\tvalid_0's error: 0.194329\n",
      "[412]\tvalid_0's error: 0.194329\n",
      "[413]\tvalid_0's error: 0.194329\n",
      "[414]\tvalid_0's error: 0.194329\n",
      "[415]\tvalid_0's error: 0.194329\n",
      "[416]\tvalid_0's error: 0.194329\n",
      "[417]\tvalid_0's error: 0.194329\n",
      "[418]\tvalid_0's error: 0.194329\n",
      "[419]\tvalid_0's error: 0.194329\n",
      "[420]\tvalid_0's error: 0.194329\n",
      "[421]\tvalid_0's error: 0.194329\n",
      "[422]\tvalid_0's error: 0.194329\n",
      "[423]\tvalid_0's error: 0.194329\n",
      "[424]\tvalid_0's error: 0.194329\n",
      "[425]\tvalid_0's error: 0.194329\n",
      "[426]\tvalid_0's error: 0.194329\n",
      "[427]\tvalid_0's error: 0.194329\n",
      "[428]\tvalid_0's error: 0.194329\n",
      "[429]\tvalid_0's error: 0.194329\n",
      "[430]\tvalid_0's error: 0.194329\n",
      "[431]\tvalid_0's error: 0.194329\n",
      "[432]\tvalid_0's error: 0.194329\n",
      "[433]\tvalid_0's error: 0.194329\n",
      "[434]\tvalid_0's error: 0.194329\n",
      "[435]\tvalid_0's error: 0.194329\n",
      "[436]\tvalid_0's error: 0.194329\n",
      "[437]\tvalid_0's error: 0.194329\n",
      "[438]\tvalid_0's error: 0.194329\n",
      "[439]\tvalid_0's error: 0.194329\n",
      "[440]\tvalid_0's error: 0.194329\n",
      "[441]\tvalid_0's error: 0.194329\n",
      "[442]\tvalid_0's error: 0.194329\n",
      "[443]\tvalid_0's error: 0.194329\n",
      "[444]\tvalid_0's error: 0.194329\n",
      "[445]\tvalid_0's error: 0.194329\n",
      "[446]\tvalid_0's error: 0.194329\n",
      "[447]\tvalid_0's error: 0.194329\n",
      "[448]\tvalid_0's error: 0.194329\n",
      "[449]\tvalid_0's error: 0.194329\n",
      "[450]\tvalid_0's error: 0.194329\n",
      "[451]\tvalid_0's error: 0.193912\n",
      "[452]\tvalid_0's error: 0.193912\n",
      "[453]\tvalid_0's error: 0.193912\n",
      "[454]\tvalid_0's error: 0.193912\n",
      "[455]\tvalid_0's error: 0.193912\n",
      "[456]\tvalid_0's error: 0.193912\n",
      "[457]\tvalid_0's error: 0.193912\n",
      "[458]\tvalid_0's error: 0.193912\n",
      "[459]\tvalid_0's error: 0.193912\n",
      "[460]\tvalid_0's error: 0.193912\n",
      "[461]\tvalid_0's error: 0.193912\n",
      "[462]\tvalid_0's error: 0.193912\n",
      "[463]\tvalid_0's error: 0.193912\n",
      "[464]\tvalid_0's error: 0.193912\n",
      "[465]\tvalid_0's error: 0.193912\n",
      "[466]\tvalid_0's error: 0.193912\n",
      "[467]\tvalid_0's error: 0.193912\n",
      "[468]\tvalid_0's error: 0.193912\n",
      "[469]\tvalid_0's error: 0.193912\n",
      "[470]\tvalid_0's error: 0.193912\n",
      "[471]\tvalid_0's error: 0.193912\n",
      "[472]\tvalid_0's error: 0.193912\n",
      "[473]\tvalid_0's error: 0.193912\n",
      "[474]\tvalid_0's error: 0.193912\n",
      "[475]\tvalid_0's error: 0.193912\n",
      "[476]\tvalid_0's error: 0.193912\n",
      "[477]\tvalid_0's error: 0.193912\n",
      "[478]\tvalid_0's error: 0.193912\n",
      "[479]\tvalid_0's error: 0.193912\n",
      "[480]\tvalid_0's error: 0.193912\n",
      "[481]\tvalid_0's error: 0.194329\n",
      "[482]\tvalid_0's error: 0.194329\n",
      "[483]\tvalid_0's error: 0.194329\n",
      "[484]\tvalid_0's error: 0.194329\n",
      "[485]\tvalid_0's error: 0.194329\n",
      "[486]\tvalid_0's error: 0.194329\n",
      "[487]\tvalid_0's error: 0.194329\n",
      "[488]\tvalid_0's error: 0.194329\n",
      "[489]\tvalid_0's error: 0.194329\n",
      "[490]\tvalid_0's error: 0.194329\n",
      "[491]\tvalid_0's error: 0.194329\n",
      "[492]\tvalid_0's error: 0.194329\n",
      "[493]\tvalid_0's error: 0.194329\n",
      "[494]\tvalid_0's error: 0.194329\n",
      "[495]\tvalid_0's error: 0.194329\n",
      "[496]\tvalid_0's error: 0.194329\n",
      "[497]\tvalid_0's error: 0.194329\n",
      "[498]\tvalid_0's error: 0.194329\n",
      "[499]\tvalid_0's error: 0.194329\n",
      "[500]\tvalid_0's error: 0.194329\n",
      "[501]\tvalid_0's error: 0.194329\n",
      "[502]\tvalid_0's error: 0.194329\n",
      "[503]\tvalid_0's error: 0.194329\n",
      "[504]\tvalid_0's error: 0.194329\n",
      "[505]\tvalid_0's error: 0.194329\n",
      "[506]\tvalid_0's error: 0.194329\n",
      "[507]\tvalid_0's error: 0.194329\n",
      "[508]\tvalid_0's error: 0.194329\n",
      "[509]\tvalid_0's error: 0.194329\n",
      "[510]\tvalid_0's error: 0.194329\n",
      "[511]\tvalid_0's error: 0.194329\n",
      "[512]\tvalid_0's error: 0.194329\n",
      "[513]\tvalid_0's error: 0.194329\n",
      "[514]\tvalid_0's error: 0.194329\n",
      "[515]\tvalid_0's error: 0.194329\n",
      "[516]\tvalid_0's error: 0.194329\n",
      "[517]\tvalid_0's error: 0.194329\n",
      "[518]\tvalid_0's error: 0.194329\n",
      "[519]\tvalid_0's error: 0.194329\n",
      "[520]\tvalid_0's error: 0.194329\n",
      "[521]\tvalid_0's error: 0.194329\n",
      "[522]\tvalid_0's error: 0.194329\n",
      "[523]\tvalid_0's error: 0.194329\n",
      "[524]\tvalid_0's error: 0.194329\n",
      "[525]\tvalid_0's error: 0.194329\n",
      "[526]\tvalid_0's error: 0.194329\n",
      "[527]\tvalid_0's error: 0.194329\n",
      "[528]\tvalid_0's error: 0.194329\n",
      "[529]\tvalid_0's error: 0.194329\n",
      "[530]\tvalid_0's error: 0.194329\n",
      "[531]\tvalid_0's error: 0.194329\n",
      "[532]\tvalid_0's error: 0.194329\n",
      "[533]\tvalid_0's error: 0.194329\n",
      "[534]\tvalid_0's error: 0.194329\n",
      "[535]\tvalid_0's error: 0.194329\n",
      "[536]\tvalid_0's error: 0.194329\n",
      "[537]\tvalid_0's error: 0.194329\n",
      "[538]\tvalid_0's error: 0.194329\n",
      "[539]\tvalid_0's error: 0.194329\n",
      "[540]\tvalid_0's error: 0.194329\n",
      "[541]\tvalid_0's error: 0.194329\n",
      "[542]\tvalid_0's error: 0.194329\n",
      "[543]\tvalid_0's error: 0.194329\n",
      "[544]\tvalid_0's error: 0.194329\n",
      "[545]\tvalid_0's error: 0.194329\n",
      "[546]\tvalid_0's error: 0.194329\n",
      "[547]\tvalid_0's error: 0.194329\n",
      "[548]\tvalid_0's error: 0.194329\n",
      "[549]\tvalid_0's error: 0.194329\n",
      "[550]\tvalid_0's error: 0.194329\n",
      "[551]\tvalid_0's error: 0.194329\n",
      "[552]\tvalid_0's error: 0.194329\n",
      "[553]\tvalid_0's error: 0.194329\n",
      "[554]\tvalid_0's error: 0.194329\n",
      "[555]\tvalid_0's error: 0.193912\n",
      "[556]\tvalid_0's error: 0.193912\n",
      "[557]\tvalid_0's error: 0.193912\n",
      "[558]\tvalid_0's error: 0.193912\n",
      "[559]\tvalid_0's error: 0.193912\n",
      "[560]\tvalid_0's error: 0.193912\n",
      "[561]\tvalid_0's error: 0.193912\n",
      "[562]\tvalid_0's error: 0.193912\n",
      "[563]\tvalid_0's error: 0.193912\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[564]\tvalid_0's error: 0.193912\n",
      "[565]\tvalid_0's error: 0.193912\n",
      "[566]\tvalid_0's error: 0.193912\n",
      "[567]\tvalid_0's error: 0.193912\n",
      "[568]\tvalid_0's error: 0.193912\n",
      "[569]\tvalid_0's error: 0.193912\n",
      "[570]\tvalid_0's error: 0.193912\n",
      "[571]\tvalid_0's error: 0.193912\n",
      "[572]\tvalid_0's error: 0.193912\n",
      "[573]\tvalid_0's error: 0.193495\n",
      "[574]\tvalid_0's error: 0.193495\n",
      "[575]\tvalid_0's error: 0.193495\n",
      "[576]\tvalid_0's error: 0.193495\n",
      "[577]\tvalid_0's error: 0.193495\n",
      "[578]\tvalid_0's error: 0.193495\n",
      "[579]\tvalid_0's error: 0.193495\n",
      "[580]\tvalid_0's error: 0.193495\n",
      "[581]\tvalid_0's error: 0.193495\n",
      "[582]\tvalid_0's error: 0.193495\n",
      "[583]\tvalid_0's error: 0.193495\n",
      "[584]\tvalid_0's error: 0.193495\n",
      "[585]\tvalid_0's error: 0.193495\n",
      "[586]\tvalid_0's error: 0.193495\n",
      "[587]\tvalid_0's error: 0.193495\n",
      "[588]\tvalid_0's error: 0.193495\n",
      "[589]\tvalid_0's error: 0.193495\n",
      "[590]\tvalid_0's error: 0.193495\n",
      "[591]\tvalid_0's error: 0.193495\n",
      "[592]\tvalid_0's error: 0.193495\n",
      "[593]\tvalid_0's error: 0.193495\n",
      "[594]\tvalid_0's error: 0.193495\n",
      "[595]\tvalid_0's error: 0.193495\n",
      "[596]\tvalid_0's error: 0.193495\n",
      "[597]\tvalid_0's error: 0.193495\n",
      "[598]\tvalid_0's error: 0.193495\n",
      "[599]\tvalid_0's error: 0.193495\n",
      "[600]\tvalid_0's error: 0.193495\n",
      "[601]\tvalid_0's error: 0.193495\n",
      "[602]\tvalid_0's error: 0.193495\n",
      "[603]\tvalid_0's error: 0.193912\n",
      "[604]\tvalid_0's error: 0.193912\n",
      "[605]\tvalid_0's error: 0.193912\n",
      "[606]\tvalid_0's error: 0.193912\n",
      "[607]\tvalid_0's error: 0.193912\n",
      "[608]\tvalid_0's error: 0.193912\n",
      "[609]\tvalid_0's error: 0.193912\n",
      "[610]\tvalid_0's error: 0.193912\n",
      "[611]\tvalid_0's error: 0.193912\n",
      "[612]\tvalid_0's error: 0.193912\n",
      "[613]\tvalid_0's error: 0.193912\n",
      "[614]\tvalid_0's error: 0.193912\n",
      "[615]\tvalid_0's error: 0.193912\n",
      "[616]\tvalid_0's error: 0.193912\n",
      "[617]\tvalid_0's error: 0.193912\n",
      "[618]\tvalid_0's error: 0.193912\n",
      "[619]\tvalid_0's error: 0.193912\n",
      "[620]\tvalid_0's error: 0.193912\n",
      "[621]\tvalid_0's error: 0.193912\n",
      "[622]\tvalid_0's error: 0.193912\n",
      "[623]\tvalid_0's error: 0.193912\n",
      "[624]\tvalid_0's error: 0.193912\n",
      "[625]\tvalid_0's error: 0.193912\n",
      "[626]\tvalid_0's error: 0.193912\n",
      "[627]\tvalid_0's error: 0.193912\n",
      "[628]\tvalid_0's error: 0.193495\n",
      "[629]\tvalid_0's error: 0.193495\n",
      "[630]\tvalid_0's error: 0.193912\n",
      "[631]\tvalid_0's error: 0.193912\n",
      "[632]\tvalid_0's error: 0.193912\n",
      "[633]\tvalid_0's error: 0.193495\n",
      "[634]\tvalid_0's error: 0.193495\n",
      "[635]\tvalid_0's error: 0.193495\n",
      "[636]\tvalid_0's error: 0.193495\n",
      "[637]\tvalid_0's error: 0.193495\n",
      "[638]\tvalid_0's error: 0.193495\n",
      "[639]\tvalid_0's error: 0.193495\n",
      "[640]\tvalid_0's error: 0.193078\n",
      "[641]\tvalid_0's error: 0.193078\n",
      "[642]\tvalid_0's error: 0.193078\n",
      "[643]\tvalid_0's error: 0.193078\n",
      "[644]\tvalid_0's error: 0.193495\n",
      "[645]\tvalid_0's error: 0.193495\n",
      "[646]\tvalid_0's error: 0.193495\n",
      "[647]\tvalid_0's error: 0.193495\n",
      "[648]\tvalid_0's error: 0.193495\n",
      "[649]\tvalid_0's error: 0.193078\n",
      "[650]\tvalid_0's error: 0.193495\n",
      "[651]\tvalid_0's error: 0.193495\n",
      "[652]\tvalid_0's error: 0.193495\n",
      "[653]\tvalid_0's error: 0.193495\n",
      "[654]\tvalid_0's error: 0.193495\n",
      "[655]\tvalid_0's error: 0.193495\n",
      "[656]\tvalid_0's error: 0.193495\n",
      "[657]\tvalid_0's error: 0.193495\n",
      "[658]\tvalid_0's error: 0.193495\n",
      "[659]\tvalid_0's error: 0.193495\n",
      "[660]\tvalid_0's error: 0.193495\n",
      "[661]\tvalid_0's error: 0.193495\n",
      "[662]\tvalid_0's error: 0.193495\n",
      "[663]\tvalid_0's error: 0.193495\n",
      "[664]\tvalid_0's error: 0.193495\n",
      "[665]\tvalid_0's error: 0.193495\n",
      "[666]\tvalid_0's error: 0.193495\n",
      "[667]\tvalid_0's error: 0.193495\n",
      "[668]\tvalid_0's error: 0.193495\n",
      "[669]\tvalid_0's error: 0.193495\n",
      "[670]\tvalid_0's error: 0.193495\n",
      "[671]\tvalid_0's error: 0.193495\n",
      "[672]\tvalid_0's error: 0.193495\n",
      "[673]\tvalid_0's error: 0.193495\n",
      "[674]\tvalid_0's error: 0.193495\n",
      "[675]\tvalid_0's error: 0.193495\n",
      "[676]\tvalid_0's error: 0.193495\n",
      "[677]\tvalid_0's error: 0.193495\n",
      "[678]\tvalid_0's error: 0.193912\n",
      "[679]\tvalid_0's error: 0.193912\n",
      "[680]\tvalid_0's error: 0.193912\n",
      "[681]\tvalid_0's error: 0.193495\n",
      "[682]\tvalid_0's error: 0.193078\n",
      "[683]\tvalid_0's error: 0.193078\n",
      "[684]\tvalid_0's error: 0.193078\n",
      "[685]\tvalid_0's error: 0.193495\n",
      "[686]\tvalid_0's error: 0.193495\n",
      "[687]\tvalid_0's error: 0.193495\n",
      "[688]\tvalid_0's error: 0.193495\n",
      "[689]\tvalid_0's error: 0.193495\n",
      "[690]\tvalid_0's error: 0.193078\n",
      "[691]\tvalid_0's error: 0.193078\n",
      "[692]\tvalid_0's error: 0.193078\n",
      "[693]\tvalid_0's error: 0.193078\n",
      "[694]\tvalid_0's error: 0.193495\n",
      "[695]\tvalid_0's error: 0.193495\n",
      "[696]\tvalid_0's error: 0.193495\n",
      "[697]\tvalid_0's error: 0.193495\n",
      "[698]\tvalid_0's error: 0.193495\n",
      "[699]\tvalid_0's error: 0.193495\n",
      "[700]\tvalid_0's error: 0.193495\n",
      "[701]\tvalid_0's error: 0.193495\n",
      "[702]\tvalid_0's error: 0.193495\n",
      "[703]\tvalid_0's error: 0.193495\n",
      "[704]\tvalid_0's error: 0.193495\n",
      "[705]\tvalid_0's error: 0.193495\n",
      "[706]\tvalid_0's error: 0.193495\n",
      "[707]\tvalid_0's error: 0.193495\n",
      "[708]\tvalid_0's error: 0.193495\n",
      "[709]\tvalid_0's error: 0.193495\n",
      "[710]\tvalid_0's error: 0.193495\n",
      "[711]\tvalid_0's error: 0.193495\n",
      "[712]\tvalid_0's error: 0.193495\n",
      "[713]\tvalid_0's error: 0.193495\n",
      "[714]\tvalid_0's error: 0.193495\n",
      "[715]\tvalid_0's error: 0.193495\n",
      "[716]\tvalid_0's error: 0.193495\n",
      "[717]\tvalid_0's error: 0.193495\n",
      "[718]\tvalid_0's error: 0.193495\n",
      "[719]\tvalid_0's error: 0.193495\n",
      "[720]\tvalid_0's error: 0.193495\n",
      "[721]\tvalid_0's error: 0.193495\n",
      "[722]\tvalid_0's error: 0.193495\n",
      "[723]\tvalid_0's error: 0.193495\n",
      "[724]\tvalid_0's error: 0.193495\n",
      "[725]\tvalid_0's error: 0.193495\n",
      "[726]\tvalid_0's error: 0.193495\n",
      "[727]\tvalid_0's error: 0.193495\n",
      "[728]\tvalid_0's error: 0.193495\n",
      "[729]\tvalid_0's error: 0.193495\n",
      "[730]\tvalid_0's error: 0.193495\n",
      "[731]\tvalid_0's error: 0.193495\n",
      "[732]\tvalid_0's error: 0.193495\n",
      "[733]\tvalid_0's error: 0.193495\n",
      "[734]\tvalid_0's error: 0.193495\n",
      "[735]\tvalid_0's error: 0.193495\n",
      "[736]\tvalid_0's error: 0.193495\n",
      "[737]\tvalid_0's error: 0.193495\n",
      "[738]\tvalid_0's error: 0.193495\n",
      "[739]\tvalid_0's error: 0.193495\n",
      "[740]\tvalid_0's error: 0.193495\n",
      "[741]\tvalid_0's error: 0.193495\n",
      "[742]\tvalid_0's error: 0.193495\n",
      "[743]\tvalid_0's error: 0.193495\n",
      "[744]\tvalid_0's error: 0.193078\n",
      "[745]\tvalid_0's error: 0.193078\n",
      "[746]\tvalid_0's error: 0.193078\n",
      "[747]\tvalid_0's error: 0.193078\n",
      "[748]\tvalid_0's error: 0.193078\n",
      "[749]\tvalid_0's error: 0.193495\n",
      "[750]\tvalid_0's error: 0.193495\n",
      "[751]\tvalid_0's error: 0.193495\n",
      "[752]\tvalid_0's error: 0.193495\n",
      "[753]\tvalid_0's error: 0.193078\n",
      "[754]\tvalid_0's error: 0.193078\n",
      "[755]\tvalid_0's error: 0.193078\n",
      "[756]\tvalid_0's error: 0.193078\n",
      "[757]\tvalid_0's error: 0.193078\n",
      "[758]\tvalid_0's error: 0.193078\n",
      "[759]\tvalid_0's error: 0.193078\n",
      "[760]\tvalid_0's error: 0.193078\n",
      "[761]\tvalid_0's error: 0.193078\n",
      "[762]\tvalid_0's error: 0.193078\n",
      "[763]\tvalid_0's error: 0.193078\n",
      "[764]\tvalid_0's error: 0.193078\n",
      "[765]\tvalid_0's error: 0.193078\n",
      "[766]\tvalid_0's error: 0.193078\n",
      "[767]\tvalid_0's error: 0.193078\n",
      "[768]\tvalid_0's error: 0.193078\n",
      "[769]\tvalid_0's error: 0.193078\n",
      "[770]\tvalid_0's error: 0.193078\n",
      "[771]\tvalid_0's error: 0.193078\n",
      "[772]\tvalid_0's error: 0.193078\n",
      "[773]\tvalid_0's error: 0.193078\n",
      "[774]\tvalid_0's error: 0.193078\n",
      "[775]\tvalid_0's error: 0.193078\n",
      "[776]\tvalid_0's error: 0.193078\n",
      "[777]\tvalid_0's error: 0.193078\n",
      "[778]\tvalid_0's error: 0.193078\n",
      "[779]\tvalid_0's error: 0.193078\n",
      "[780]\tvalid_0's error: 0.193078\n",
      "[781]\tvalid_0's error: 0.193078\n",
      "[782]\tvalid_0's error: 0.193078\n",
      "[783]\tvalid_0's error: 0.193078\n",
      "[784]\tvalid_0's error: 0.193078\n",
      "[785]\tvalid_0's error: 0.193078\n",
      "[786]\tvalid_0's error: 0.193078\n",
      "[787]\tvalid_0's error: 0.193078\n",
      "[788]\tvalid_0's error: 0.193078\n",
      "[789]\tvalid_0's error: 0.193078\n",
      "[790]\tvalid_0's error: 0.193078\n",
      "[791]\tvalid_0's error: 0.193078\n",
      "[792]\tvalid_0's error: 0.193078\n",
      "[793]\tvalid_0's error: 0.193078\n",
      "[794]\tvalid_0's error: 0.193078\n",
      "[795]\tvalid_0's error: 0.193078\n",
      "[796]\tvalid_0's error: 0.193078\n",
      "[797]\tvalid_0's error: 0.193078\n",
      "[798]\tvalid_0's error: 0.193078\n",
      "[799]\tvalid_0's error: 0.193078\n",
      "[800]\tvalid_0's error: 0.193078\n",
      "[801]\tvalid_0's error: 0.193078\n",
      "[802]\tvalid_0's error: 0.193078\n",
      "[803]\tvalid_0's error: 0.193078\n",
      "[804]\tvalid_0's error: 0.193078\n",
      "[805]\tvalid_0's error: 0.193078\n",
      "[806]\tvalid_0's error: 0.193078\n",
      "[807]\tvalid_0's error: 0.193078\n",
      "[808]\tvalid_0's error: 0.193078\n",
      "[809]\tvalid_0's error: 0.193078\n",
      "[810]\tvalid_0's error: 0.193078\n",
      "[811]\tvalid_0's error: 0.193078\n",
      "[812]\tvalid_0's error: 0.193078\n",
      "[813]\tvalid_0's error: 0.193078\n",
      "[814]\tvalid_0's error: 0.193078\n",
      "[815]\tvalid_0's error: 0.193078\n",
      "[816]\tvalid_0's error: 0.193078\n",
      "[817]\tvalid_0's error: 0.193078\n",
      "[818]\tvalid_0's error: 0.193078\n",
      "[819]\tvalid_0's error: 0.193078\n",
      "[820]\tvalid_0's error: 0.193078\n",
      "[821]\tvalid_0's error: 0.193078\n",
      "[822]\tvalid_0's error: 0.193078\n",
      "[823]\tvalid_0's error: 0.193078\n",
      "[824]\tvalid_0's error: 0.193078\n",
      "[825]\tvalid_0's error: 0.193078\n",
      "[826]\tvalid_0's error: 0.193078\n",
      "[827]\tvalid_0's error: 0.193078\n",
      "[828]\tvalid_0's error: 0.193078\n",
      "[829]\tvalid_0's error: 0.193078\n",
      "[830]\tvalid_0's error: 0.193078\n",
      "[831]\tvalid_0's error: 0.193078\n",
      "[832]\tvalid_0's error: 0.193078\n",
      "[833]\tvalid_0's error: 0.192661\n",
      "[834]\tvalid_0's error: 0.192661\n",
      "[835]\tvalid_0's error: 0.192661\n",
      "[836]\tvalid_0's error: 0.192661\n",
      "[837]\tvalid_0's error: 0.192661\n",
      "[838]\tvalid_0's error: 0.192661\n",
      "[839]\tvalid_0's error: 0.192661\n",
      "[840]\tvalid_0's error: 0.192661\n",
      "[841]\tvalid_0's error: 0.192661\n",
      "[842]\tvalid_0's error: 0.192661\n",
      "[843]\tvalid_0's error: 0.192661\n",
      "[844]\tvalid_0's error: 0.192661\n",
      "[845]\tvalid_0's error: 0.192661\n",
      "[846]\tvalid_0's error: 0.192661\n",
      "[847]\tvalid_0's error: 0.192661\n",
      "[848]\tvalid_0's error: 0.192661\n",
      "[849]\tvalid_0's error: 0.192661\n",
      "[850]\tvalid_0's error: 0.192661\n",
      "[851]\tvalid_0's error: 0.192661\n",
      "[852]\tvalid_0's error: 0.192661\n",
      "[853]\tvalid_0's error: 0.192661\n",
      "[854]\tvalid_0's error: 0.192661\n",
      "[855]\tvalid_0's error: 0.192661\n",
      "[856]\tvalid_0's error: 0.192661\n",
      "[857]\tvalid_0's error: 0.192661\n",
      "[858]\tvalid_0's error: 0.192661\n",
      "[859]\tvalid_0's error: 0.192661\n",
      "[860]\tvalid_0's error: 0.192661\n",
      "[861]\tvalid_0's error: 0.192661\n",
      "[862]\tvalid_0's error: 0.192661\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[863]\tvalid_0's error: 0.192661\n",
      "[864]\tvalid_0's error: 0.192661\n",
      "[865]\tvalid_0's error: 0.192661\n",
      "[866]\tvalid_0's error: 0.192661\n",
      "[867]\tvalid_0's error: 0.192661\n",
      "[868]\tvalid_0's error: 0.192661\n",
      "[869]\tvalid_0's error: 0.192661\n",
      "[870]\tvalid_0's error: 0.192661\n",
      "[871]\tvalid_0's error: 0.192661\n",
      "[872]\tvalid_0's error: 0.192661\n",
      "[873]\tvalid_0's error: 0.192661\n",
      "[874]\tvalid_0's error: 0.192661\n",
      "[875]\tvalid_0's error: 0.192661\n",
      "[876]\tvalid_0's error: 0.193078\n",
      "[877]\tvalid_0's error: 0.193078\n",
      "[878]\tvalid_0's error: 0.193078\n",
      "[879]\tvalid_0's error: 0.193078\n",
      "[880]\tvalid_0's error: 0.193078\n",
      "[881]\tvalid_0's error: 0.193078\n",
      "[882]\tvalid_0's error: 0.193078\n",
      "[883]\tvalid_0's error: 0.193078\n",
      "[884]\tvalid_0's error: 0.193078\n",
      "[885]\tvalid_0's error: 0.193078\n",
      "[886]\tvalid_0's error: 0.193078\n",
      "[887]\tvalid_0's error: 0.193078\n",
      "[888]\tvalid_0's error: 0.193078\n",
      "[889]\tvalid_0's error: 0.193078\n",
      "[890]\tvalid_0's error: 0.192661\n",
      "[891]\tvalid_0's error: 0.192661\n",
      "[892]\tvalid_0's error: 0.192661\n",
      "[893]\tvalid_0's error: 0.192661\n",
      "[894]\tvalid_0's error: 0.192661\n",
      "[895]\tvalid_0's error: 0.192661\n",
      "[896]\tvalid_0's error: 0.192661\n",
      "[897]\tvalid_0's error: 0.192661\n",
      "[898]\tvalid_0's error: 0.192661\n",
      "[899]\tvalid_0's error: 0.192661\n",
      "[900]\tvalid_0's error: 0.192661\n",
      "[901]\tvalid_0's error: 0.192661\n",
      "[902]\tvalid_0's error: 0.192661\n",
      "[903]\tvalid_0's error: 0.192661\n",
      "[904]\tvalid_0's error: 0.192661\n",
      "[905]\tvalid_0's error: 0.192661\n",
      "[906]\tvalid_0's error: 0.192244\n",
      "[907]\tvalid_0's error: 0.192244\n",
      "[908]\tvalid_0's error: 0.192244\n",
      "[909]\tvalid_0's error: 0.192244\n",
      "[910]\tvalid_0's error: 0.192244\n",
      "[911]\tvalid_0's error: 0.192244\n",
      "[912]\tvalid_0's error: 0.192244\n",
      "[913]\tvalid_0's error: 0.192244\n",
      "[914]\tvalid_0's error: 0.192244\n",
      "[915]\tvalid_0's error: 0.192244\n",
      "[916]\tvalid_0's error: 0.192244\n",
      "[917]\tvalid_0's error: 0.192244\n",
      "[918]\tvalid_0's error: 0.192244\n",
      "[919]\tvalid_0's error: 0.192244\n",
      "[920]\tvalid_0's error: 0.192244\n",
      "[921]\tvalid_0's error: 0.192244\n",
      "[922]\tvalid_0's error: 0.192244\n",
      "[923]\tvalid_0's error: 0.192244\n",
      "[924]\tvalid_0's error: 0.192244\n",
      "[925]\tvalid_0's error: 0.192244\n",
      "[926]\tvalid_0's error: 0.192244\n",
      "[927]\tvalid_0's error: 0.192244\n",
      "[928]\tvalid_0's error: 0.192244\n",
      "[929]\tvalid_0's error: 0.192244\n",
      "[930]\tvalid_0's error: 0.191827\n",
      "[931]\tvalid_0's error: 0.191827\n",
      "[932]\tvalid_0's error: 0.190992\n",
      "[933]\tvalid_0's error: 0.190992\n",
      "[934]\tvalid_0's error: 0.190992\n",
      "[935]\tvalid_0's error: 0.190992\n",
      "[936]\tvalid_0's error: 0.190992\n",
      "[937]\tvalid_0's error: 0.190992\n",
      "[938]\tvalid_0's error: 0.190992\n",
      "[939]\tvalid_0's error: 0.190992\n",
      "[940]\tvalid_0's error: 0.190992\n",
      "[941]\tvalid_0's error: 0.190992\n",
      "[942]\tvalid_0's error: 0.190992\n",
      "[943]\tvalid_0's error: 0.190992\n",
      "[944]\tvalid_0's error: 0.190575\n",
      "[945]\tvalid_0's error: 0.190575\n",
      "[946]\tvalid_0's error: 0.190575\n",
      "[947]\tvalid_0's error: 0.190575\n",
      "[948]\tvalid_0's error: 0.190575\n",
      "[949]\tvalid_0's error: 0.190575\n",
      "[950]\tvalid_0's error: 0.190575\n",
      "[951]\tvalid_0's error: 0.190575\n",
      "[952]\tvalid_0's error: 0.190575\n",
      "[953]\tvalid_0's error: 0.190575\n",
      "[954]\tvalid_0's error: 0.190575\n",
      "[955]\tvalid_0's error: 0.190575\n",
      "[956]\tvalid_0's error: 0.190575\n",
      "[957]\tvalid_0's error: 0.190575\n",
      "[958]\tvalid_0's error: 0.190575\n",
      "[959]\tvalid_0's error: 0.190575\n",
      "[960]\tvalid_0's error: 0.190575\n",
      "[961]\tvalid_0's error: 0.190575\n",
      "[962]\tvalid_0's error: 0.190575\n",
      "[963]\tvalid_0's error: 0.190575\n",
      "[964]\tvalid_0's error: 0.190575\n",
      "[965]\tvalid_0's error: 0.190575\n",
      "[966]\tvalid_0's error: 0.190575\n",
      "[967]\tvalid_0's error: 0.190575\n",
      "[968]\tvalid_0's error: 0.190575\n",
      "[969]\tvalid_0's error: 0.190158\n",
      "[970]\tvalid_0's error: 0.190158\n",
      "[971]\tvalid_0's error: 0.190575\n",
      "[972]\tvalid_0's error: 0.190575\n",
      "[973]\tvalid_0's error: 0.190575\n",
      "[974]\tvalid_0's error: 0.190158\n",
      "[975]\tvalid_0's error: 0.190158\n",
      "[976]\tvalid_0's error: 0.190158\n",
      "[977]\tvalid_0's error: 0.190158\n",
      "[978]\tvalid_0's error: 0.190158\n",
      "[979]\tvalid_0's error: 0.190158\n",
      "[980]\tvalid_0's error: 0.190158\n",
      "[981]\tvalid_0's error: 0.190158\n",
      "[982]\tvalid_0's error: 0.190575\n",
      "[983]\tvalid_0's error: 0.190575\n",
      "[984]\tvalid_0's error: 0.190992\n",
      "[985]\tvalid_0's error: 0.190575\n",
      "[986]\tvalid_0's error: 0.190575\n",
      "[987]\tvalid_0's error: 0.190575\n",
      "[988]\tvalid_0's error: 0.190575\n",
      "[989]\tvalid_0's error: 0.190575\n",
      "[990]\tvalid_0's error: 0.190575\n",
      "[991]\tvalid_0's error: 0.190575\n",
      "[992]\tvalid_0's error: 0.190575\n",
      "[993]\tvalid_0's error: 0.19141\n",
      "[994]\tvalid_0's error: 0.19141\n",
      "[995]\tvalid_0's error: 0.190992\n",
      "[996]\tvalid_0's error: 0.190992\n",
      "[997]\tvalid_0's error: 0.190992\n",
      "[998]\tvalid_0's error: 0.190992\n",
      "[999]\tvalid_0's error: 0.190992\n",
      "[1000]\tvalid_0's error: 0.190992\n"
     ]
    }
   ],
   "source": [
    "#!/usr/bin/env python\n",
    "# -*- coding: utf-8 -*-\n",
    "# @File  : LGB自定义损失函数.py\n",
    "# @Author: Richard Chiming Xu\n",
    "# @Date  : 2022/1/12\n",
    "# @Desc  :\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn import datasets\n",
    "\n",
    "from sklearn.utils import shuffle\n",
    "from sklearn import metrics\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from sklearn.model_selection import train_test_split\n",
    "import lightgbm as lgb\n",
    "\n",
    "# 画图\n",
    "import matplotlib.pyplot as plt\n",
    "# 参数搜索\n",
    "from sklearn.model_selection import GridSearchCV, RandomizedSearchCV, cross_val_score\n",
    "from bayes_opt import BayesianOptimization\n",
    "\n",
    "import json\n",
    "import pickle\n",
    "from tqdm import tqdm\n",
    "\n",
    "import warnings\n",
    "\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "# 读取数据\n",
    "data = pd.read_csv(\"https://cdn.coggle.club/kaggle-flight-delays/flights_10k.csv.zip\")\n",
    "\n",
    "# 提取有用的列\n",
    "data = data[[\"MONTH\", \"DAY\", \"DAY_OF_WEEK\", \"AIRLINE\", \"FLIGHT_NUMBER\", \"DESTINATION_AIRPORT\",\n",
    "             \"ORIGIN_AIRPORT\", \"AIR_TIME\", \"DEPARTURE_TIME\", \"DISTANCE\", \"ARRIVAL_DELAY\"]]\n",
    "data.dropna(inplace=True)\n",
    "\n",
    "# 筛选出部分数据\n",
    "data[\"ARRIVAL_DELAY\"] = (data[\"ARRIVAL_DELAY\"] > 10) * 1\n",
    "\n",
    "# 进行编码\n",
    "cols = [\"AIRLINE\", \"FLIGHT_NUMBER\", \"DESTINATION_AIRPORT\", \"ORIGIN_AIRPORT\"]\n",
    "for item in cols:\n",
    "    data[item] = data[item].astype(\"category\").cat.codes + 1\n",
    "\n",
    "# 划分训练集和测试集\n",
    "train, test, y_train, y_test = train_test_split(data.drop([\"ARRIVAL_DELAY\"], axis=1), data[\"ARRIVAL_DELAY\"],\n",
    "                                                random_state=10, test_size=0.25)\n",
    "\n",
    "\n",
    "def loglikelihood(preds, train_data):\n",
    "    labels = train_data.get_label()\n",
    "    preds = 1. / (1. + np.exp(-preds))\n",
    "    grad = preds - labels\n",
    "    hess = preds * (1. - preds)\n",
    "\n",
    "    proba = preds[np.array(labels) == 1]\n",
    "    if len(proba[proba < 0.1]) > 0:\n",
    "        grad *= 2\n",
    "    return grad, hess\n",
    "\n",
    "\n",
    "def binary_error(preds, train_data):\n",
    "    labels = train_data.get_label()\n",
    "    preds = 1. / (1. + np.exp(-preds))\n",
    "    return 'error', np.mean(labels != (preds > 0.8)), False\n",
    "\n",
    "\n",
    "# callback中重修改模型参数\n",
    "def reset_metrics(X_test, y_test):\n",
    "    def callback(env):\n",
    "        origin_lr = env.params['learning_rate']\n",
    "        ############  无聊代码，存粹为了体验一下callback   ############\n",
    "        proba = env.model.predict(X_test)\n",
    "        \n",
    "        # 学习率衰减\n",
    "        env.params['learning_rate'] = origin_lr * 0.99**env.iteration\n",
    "        ############  无聊代码，存粹为了体验一下callback   ############\n",
    "\n",
    "    callback.before_iteration = True\n",
    "    callback.order = 0\n",
    "    return callback\n",
    "\n",
    "\n",
    "lgb_train = lgb.Dataset(train, y_train, free_raw_data=False, silent=True)\n",
    "lgb_val = lgb.Dataset(test, y_test)\n",
    "lgb_params = {\n",
    "    'learning_rate': 0.01,\n",
    "    'max_depth': 5,\n",
    "    'n_estimators': 1000,\n",
    "    'num_leaves': 128,\n",
    "    'subsample_for_bin': 1000,\n",
    "    'refit_decay_rate': 0.1,\n",
    "    'verbose': -1\n",
    "\n",
    "}\n",
    "clf = lgb.train(params=lgb_params, train_set=lgb_train, num_boost_round=200, valid_sets=lgb_val,\n",
    "                fobj=loglikelihood, feval=binary_error)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e876561a",
   "metadata": {},
   "source": [
    "# 任务9：模型部署与加速（可选，不参与积分）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "3b4108dc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[16:58:23] D:\\a\\1\\s\\src\\compiler\\ast\\split.cc:24: Parallel compilation disabled; all member trees will be dumped to a single source file. This may increase compilation time and memory usage.\n"
     ]
    }
   ],
   "source": [
    "# lgb导出模型\n",
    "clf.save_model('lgb.txt')\n",
    "from treelite import Model\n",
    "model = Model.load('lgb.txt', model_format='lightgbm')\n",
    "model.compile('lgb.so')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f3add3d0",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
